Java Code Examples for org.apache.flink.runtime.jobgraph.JobStatus#FINISHED

The following examples show how to use org.apache.flink.runtime.jobgraph.JobStatus#FINISHED . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: ArchivedExecutionGraphBuilder.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
public ArchivedExecutionGraph build() {
	JobID jobID = this.jobID != null ? this.jobID : new JobID();
	String jobName = this.jobName != null ? this.jobName : "job_" + RANDOM.nextInt();

	if (tasks == null) {
		tasks = Collections.emptyMap();
	}

	return new ArchivedExecutionGraph(
		jobID,
		jobName,
		tasks,
		verticesInCreationOrder != null ? verticesInCreationOrder : new ArrayList<>(tasks.values()),
		stateTimestamps != null ? stateTimestamps : new long[JobStatus.values().length],
		state != null ? state : JobStatus.FINISHED,
		failureCause,
		jsonPlan != null ? jsonPlan : "{\"jobid\":\"" + jobID + "\", \"name\":\"" + jobName + "\", \"nodes\":[]}",
		archivedUserAccumulators != null ? archivedUserAccumulators : new StringifiedAccumulatorResult[0],
		serializedUserAccumulators != null ? serializedUserAccumulators : Collections.emptyMap(),
		archivedExecutionConfig != null ? archivedExecutionConfig : new ArchivedExecutionConfigBuilder().build(),
		isStoppable,
		null,
		null
	);
}
Example 2
Source File: ArchivedExecutionGraphBuilder.java    From flink with Apache License 2.0 6 votes vote down vote up 
public ArchivedExecutionGraph build() {
	JobID jobID = this.jobID != null ? this.jobID : new JobID();
	String jobName = this.jobName != null ? this.jobName : "job_" + RANDOM.nextInt();

	if (tasks == null) {
		tasks = Collections.emptyMap();
	}

	return new ArchivedExecutionGraph(
		jobID,
		jobName,
		tasks,
		verticesInCreationOrder != null ? verticesInCreationOrder : new ArrayList<>(tasks.values()),
		stateTimestamps != null ? stateTimestamps : new long[JobStatus.values().length],
		state != null ? state : JobStatus.FINISHED,
		failureCause,
		jsonPlan != null ? jsonPlan : "{\"jobid\":\"" + jobID + "\", \"name\":\"" + jobName + "\", \"nodes\":[]}",
		archivedUserAccumulators != null ? archivedUserAccumulators : new StringifiedAccumulatorResult[0],
		serializedUserAccumulators != null ? serializedUserAccumulators : Collections.emptyMap(),
		archivedExecutionConfig != null ? archivedExecutionConfig : new ArchivedExecutionConfigBuilder().build(),
		isStoppable,
		null,
		null
	);
}
Example 3
Source File: CompletedCheckpoint.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
public boolean discardOnShutdown(JobStatus jobStatus) throws Exception {

		if (jobStatus == JobStatus.FINISHED && props.discardOnJobFinished() ||
				jobStatus == JobStatus.CANCELED && props.discardOnJobCancelled() ||
				jobStatus == JobStatus.FAILED && props.discardOnJobFailed() ||
				jobStatus == JobStatus.SUSPENDED && props.discardOnJobSuspended()) {

			doDiscard();
			return true;
		} else {
			LOG.info("Checkpoint with ID {} at '{}' not discarded.", checkpointID, externalPointer);
			return false;
		}
	}
Example 4
Source File: JobResult.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
/**
 * Creates the {@link JobResult} from the given {@link AccessExecutionGraph} which
 * must be in a globally terminal state.
 *
 * @param accessExecutionGraph to create the JobResult from
 * @return JobResult of the given AccessExecutionGraph
 */
public static JobResult createFrom(AccessExecutionGraph accessExecutionGraph) {
	final JobID jobId = accessExecutionGraph.getJobID();
	final JobStatus jobStatus = accessExecutionGraph.getState();

	checkArgument(
		jobStatus.isGloballyTerminalState(),
		"The job " + accessExecutionGraph.getJobName() + '(' + jobId + ") is not in a globally " +
			"terminal state. It is in state " + jobStatus + '.');

	final JobResult.Builder builder = new JobResult.Builder();
	builder.jobId(jobId);

	builder.applicationStatus(ApplicationStatus.fromJobStatus(accessExecutionGraph.getState()));

	final long netRuntime = accessExecutionGraph.getStatusTimestamp(jobStatus) - accessExecutionGraph.getStatusTimestamp(JobStatus.CREATED);
	// guard against clock changes
	final long guardedNetRuntime = Math.max(netRuntime, 0L);
	builder.netRuntime(guardedNetRuntime);
	builder.accumulatorResults(accessExecutionGraph.getAccumulatorsSerialized());

	if (jobStatus != JobStatus.FINISHED) {
		final ErrorInfo errorInfo = accessExecutionGraph.getFailureInfo();

		if (errorInfo != null) {
			builder.serializedThrowable(errorInfo.getException());
		}
	}

	return builder.build();
}
Example 5
Source File: JobExecutionResultHandlerTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Test
public void testCompletedResult() throws Exception {
	final JobStatus jobStatus = JobStatus.FINISHED;
	final ArchivedExecutionGraph executionGraph = new ArchivedExecutionGraphBuilder()
		.setJobID(TEST_JOB_ID)
		.setState(jobStatus)
		.build();

	final TestingRestfulGateway testingRestfulGateway = TestingRestfulGateway.newBuilder()
		.setRequestJobStatusFunction(
			jobId -> {
				assertThat(jobId, equalTo(TEST_JOB_ID));
				return CompletableFuture.completedFuture(jobStatus);
			})
		.setRequestJobResultFunction(
			jobId -> {
				assertThat(jobId, equalTo(TEST_JOB_ID));
				return CompletableFuture.completedFuture(JobResult.createFrom(executionGraph));
			}
		)
		.build();

	final JobExecutionResultResponseBody responseBody = jobExecutionResultHandler.handleRequest(
		testRequest,
		testingRestfulGateway).get();

	assertThat(
		responseBody.getStatus().getId(),
		equalTo(QueueStatus.Id.COMPLETED));
	assertThat(responseBody.getJobExecutionResult(), not(nullValue()));
}
Example 6
Source File: CompletedCheckpoint.java    From flink with Apache License 2.0 5 votes vote down vote up 
public boolean discardOnShutdown(JobStatus jobStatus) throws Exception {

		if (jobStatus == JobStatus.FINISHED && props.discardOnJobFinished() ||
				jobStatus == JobStatus.CANCELED && props.discardOnJobCancelled() ||
				jobStatus == JobStatus.FAILED && props.discardOnJobFailed() ||
				jobStatus == JobStatus.SUSPENDED && props.discardOnJobSuspended()) {

			doDiscard();
			return true;
		} else {
			LOG.info("Checkpoint with ID {} at '{}' not discarded.", checkpointID, externalPointer);
			return false;
		}
	}
Example 7
Source File: JobResult.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Creates the {@link JobResult} from the given {@link AccessExecutionGraph} which
 * must be in a globally terminal state.
 *
 * @param accessExecutionGraph to create the JobResult from
 * @return JobResult of the given AccessExecutionGraph
 */
public static JobResult createFrom(AccessExecutionGraph accessExecutionGraph) {
	final JobID jobId = accessExecutionGraph.getJobID();
	final JobStatus jobStatus = accessExecutionGraph.getState();

	checkArgument(
		jobStatus.isGloballyTerminalState(),
		"The job " + accessExecutionGraph.getJobName() + '(' + jobId + ") is not in a globally " +
			"terminal state. It is in state " + jobStatus + '.');

	final JobResult.Builder builder = new JobResult.Builder();
	builder.jobId(jobId);

	builder.applicationStatus(ApplicationStatus.fromJobStatus(accessExecutionGraph.getState()));

	final long netRuntime = accessExecutionGraph.getStatusTimestamp(jobStatus) - accessExecutionGraph.getStatusTimestamp(JobStatus.CREATED);
	// guard against clock changes
	final long guardedNetRuntime = Math.max(netRuntime, 0L);
	builder.netRuntime(guardedNetRuntime);
	builder.accumulatorResults(accessExecutionGraph.getAccumulatorsSerialized());

	if (jobStatus != JobStatus.FINISHED) {
		final ErrorInfo errorInfo = accessExecutionGraph.getFailureInfo();

		if (errorInfo != null) {
			builder.serializedThrowable(errorInfo.getException());
		}
	}

	return builder.build();
}
Example 8
Source File: JobExecutionResultHandlerTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testCompletedResult() throws Exception {
	final JobStatus jobStatus = JobStatus.FINISHED;
	final ArchivedExecutionGraph executionGraph = new ArchivedExecutionGraphBuilder()
		.setJobID(TEST_JOB_ID)
		.setState(jobStatus)
		.build();

	final TestingRestfulGateway testingRestfulGateway = TestingRestfulGateway.newBuilder()
		.setRequestJobStatusFunction(
			jobId -> {
				assertThat(jobId, equalTo(TEST_JOB_ID));
				return CompletableFuture.completedFuture(jobStatus);
			})
		.setRequestJobResultFunction(
			jobId -> {
				assertThat(jobId, equalTo(TEST_JOB_ID));
				return CompletableFuture.completedFuture(JobResult.createFrom(executionGraph));
			}
		)
		.build();

	final JobExecutionResultResponseBody responseBody = jobExecutionResultHandler.handleRequest(
		testRequest,
		testingRestfulGateway).get();

	assertThat(
		responseBody.getStatus().getId(),
		equalTo(QueueStatus.Id.COMPLETED));
	assertThat(responseBody.getJobExecutionResult(), not(nullValue()));
}
Example 9
Source File: CompletedCheckpointTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
/**
 * Tests that the garbage collection properties are respected when shutting down.
 */
@Test
public void testCleanUpOnShutdown() throws Exception {
	JobStatus[] terminalStates = new JobStatus[] {
			JobStatus.FINISHED, JobStatus.CANCELED, JobStatus.FAILED, JobStatus.SUSPENDED
	};

	for (JobStatus status : terminalStates) {

		OperatorState state = mock(OperatorState.class);
		Map<OperatorID, OperatorState> operatorStates = new HashMap<>();
		operatorStates.put(new OperatorID(), state);

		EmptyStreamStateHandle retainedHandle = new EmptyStreamStateHandle();
		TestCompletedCheckpointStorageLocation retainedLocation =
				new TestCompletedCheckpointStorageLocation(retainedHandle, "ptr");

		// Keep
		CheckpointProperties retainProps = new CheckpointProperties(false, CheckpointType.CHECKPOINT, false, false, false, false, false);
		CompletedCheckpoint checkpoint = new CompletedCheckpoint(
				new JobID(), 0, 0, 1,
				new HashMap<>(operatorStates),
				Collections.emptyList(),
				retainProps,
				retainedLocation);

		checkpoint.discardOnShutdown(status);

		verify(state, times(0)).discardState();
		assertFalse(retainedLocation.isDisposed());
		assertFalse(retainedHandle.isDisposed());

		// Discard
		EmptyStreamStateHandle discardHandle = new EmptyStreamStateHandle();
		TestCompletedCheckpointStorageLocation discardLocation =
				new TestCompletedCheckpointStorageLocation(discardHandle, "ptr");

		// Keep
		CheckpointProperties discardProps = new CheckpointProperties(false, CheckpointType.CHECKPOINT, true, true, true, true, true);

		checkpoint = new CompletedCheckpoint(
				new JobID(), 0, 0, 1,
				new HashMap<>(operatorStates),
				Collections.emptyList(),
				discardProps,
				discardLocation);

		checkpoint.discardOnShutdown(status);

		verify(state, times(1)).discardState();
		assertTrue(discardLocation.isDisposed());
		assertTrue(discardHandle.isDisposed());
	}
}
Example 10
Source File: MultipleJobsDetailsTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
/**
 * Tests that we can un/marshal {@link MultipleJobsDetails} objects.
 */
@Test
public void testMultipleJobsDetailsMarshalling() throws JsonProcessingException {
	int[] verticesPerState = new int[ExecutionState.values().length];

	for (int i = 0; i < verticesPerState.length; i++) {
		verticesPerState[i] = i;
	}

	final JobDetails running = new JobDetails(
		new JobID(),
		"running",
		1L,
		-1L,
		9L,
		JobStatus.RUNNING,
		9L,
		verticesPerState,
		9);

	final JobDetails finished = new JobDetails(
		new JobID(),
		"finished",
		1L,
		5L,
		4L,
		JobStatus.FINISHED,
		8L,
		verticesPerState,
		4);

	final MultipleJobsDetails expected = new MultipleJobsDetails(
		Arrays.asList(running, finished));

	final ObjectMapper objectMapper = RestMapperUtils.getStrictObjectMapper();

	final JsonNode marshalled = objectMapper.valueToTree(expected);

	final MultipleJobsDetails unmarshalled = objectMapper.treeToValue(marshalled, MultipleJobsDetails.class);

	assertEquals(expected, unmarshalled);
}
Example 11
Source File: RestClusterClientTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@Override
protected CompletableFuture<MultipleJobsDetails> handleRequest(@Nonnull HandlerRequest<EmptyRequestBody, EmptyMessageParameters> request, @Nonnull DispatcherGateway gateway) throws RestHandlerException {
	JobDetails running = new JobDetails(new JobID(), "job1", 0, 0, 0, JobStatus.RUNNING, 0, new int[9], 0);
	JobDetails finished = new JobDetails(new JobID(), "job2", 0, 0, 0, JobStatus.FINISHED, 0, new int[9], 0);
	return CompletableFuture.completedFuture(new MultipleJobsDetails(Arrays.asList(running, finished)));
}
Example 12
Source File: CompletedCheckpointTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * Tests that the garbage collection properties are respected when shutting down.
 */
@Test
public void testCleanUpOnShutdown() throws Exception {
	JobStatus[] terminalStates = new JobStatus[] {
			JobStatus.FINISHED, JobStatus.CANCELED, JobStatus.FAILED, JobStatus.SUSPENDED
	};

	for (JobStatus status : terminalStates) {

		OperatorState state = mock(OperatorState.class);
		Map<OperatorID, OperatorState> operatorStates = new HashMap<>();
		operatorStates.put(new OperatorID(), state);

		EmptyStreamStateHandle retainedHandle = new EmptyStreamStateHandle();
		TestCompletedCheckpointStorageLocation retainedLocation =
				new TestCompletedCheckpointStorageLocation(retainedHandle, "ptr");

		// Keep
		CheckpointProperties retainProps = new CheckpointProperties(false, CheckpointType.CHECKPOINT, false, false, false, false, false);
		CompletedCheckpoint checkpoint = new CompletedCheckpoint(
				new JobID(), 0, 0, 1,
				new HashMap<>(operatorStates),
				Collections.emptyList(),
				retainProps,
				retainedLocation);

		checkpoint.discardOnShutdown(status);

		verify(state, times(0)).discardState();
		assertFalse(retainedLocation.isDisposed());
		assertFalse(retainedHandle.isDisposed());

		// Discard
		EmptyStreamStateHandle discardHandle = new EmptyStreamStateHandle();
		TestCompletedCheckpointStorageLocation discardLocation =
				new TestCompletedCheckpointStorageLocation(discardHandle, "ptr");

		// Keep
		CheckpointProperties discardProps = new CheckpointProperties(false, CheckpointType.CHECKPOINT, true, true, true, true, true);

		checkpoint = new CompletedCheckpoint(
				new JobID(), 0, 0, 1,
				new HashMap<>(operatorStates),
				Collections.emptyList(),
				discardProps,
				discardLocation);

		checkpoint.discardOnShutdown(status);

		verify(state, times(1)).discardState();
		assertTrue(discardLocation.isDisposed());
		assertTrue(discardHandle.isDisposed());
	}
}
Example 13
Source File: MultipleJobsDetailsTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * Tests that we can un/marshal {@link MultipleJobsDetails} objects.
 */
@Test
public void testMultipleJobsDetailsMarshalling() throws JsonProcessingException {
	int[] verticesPerState = new int[ExecutionState.values().length];

	for (int i = 0; i < verticesPerState.length; i++) {
		verticesPerState[i] = i;
	}

	final JobDetails running = new JobDetails(
		new JobID(),
		"running",
		1L,
		-1L,
		9L,
		JobStatus.RUNNING,
		9L,
		verticesPerState,
		9);

	final JobDetails finished = new JobDetails(
		new JobID(),
		"finished",
		1L,
		5L,
		4L,
		JobStatus.FINISHED,
		8L,
		verticesPerState,
		4);

	final MultipleJobsDetails expected = new MultipleJobsDetails(
		Arrays.asList(running, finished));

	final ObjectMapper objectMapper = RestMapperUtils.getStrictObjectMapper();

	final JsonNode marshalled = objectMapper.valueToTree(expected);

	final MultipleJobsDetails unmarshalled = objectMapper.treeToValue(marshalled, MultipleJobsDetails.class);

	assertEquals(expected, unmarshalled);
}
Example 14
Source File: RestClusterClientTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Override
protected CompletableFuture<MultipleJobsDetails> handleRequest(@Nonnull HandlerRequest<EmptyRequestBody, EmptyMessageParameters> request, @Nonnull DispatcherGateway gateway) throws RestHandlerException {
	JobDetails running = new JobDetails(new JobID(), "job1", 0, 0, 0, JobStatus.RUNNING, 0, new int[9], 0);
	JobDetails finished = new JobDetails(new JobID(), "job2", 0, 0, 0, JobStatus.FINISHED, 0, new int[9], 0);
	return CompletableFuture.completedFuture(new MultipleJobsDetails(Arrays.asList(running, finished)));
}