Java Code Examples for org.apache.flink.runtime.jobgraph.JobStatus#values()

private Collection<JobDetails> randomJobDetails(Random rnd) {
	final JobDetails[] details = new JobDetails[rnd.nextInt(10)];
	for (int k = 0; k < details.length; k++) {
		int[] numVerticesPerState = new int[ExecutionState.values().length];
		int numTotal = 0;

		for (int i = 0; i < numVerticesPerState.length; i++) {
			int count = rnd.nextInt(55);
			numVerticesPerState[i] = count;
			numTotal += count;
		}

		long time = rnd.nextLong();
		long endTime = rnd.nextBoolean() ? -1L : time + rnd.nextInt();
		long lastModified = endTime == -1 ? time + rnd.nextInt() : endTime;

		String name = new GenericMessageTester.StringInstantiator().instantiate(rnd);
		JobID jid = new JobID();
		JobStatus status = JobStatus.values()[rnd.nextInt(JobStatus.values().length)];

		details[k] = new JobDetails(jid, name, time, endTime, endTime - time, status, lastModified, numVerticesPerState, numTotal);
	}
	return Arrays.asList(details);
}
 

public ArchivedExecutionGraph build() {
	JobID jobID = this.jobID != null ? this.jobID : new JobID();
	String jobName = this.jobName != null ? this.jobName : "job_" + RANDOM.nextInt();

	if (tasks == null) {
		tasks = Collections.emptyMap();
	}

	return new ArchivedExecutionGraph(
		jobID,
		jobName,
		tasks,
		verticesInCreationOrder != null ? verticesInCreationOrder : new ArrayList<>(tasks.values()),
		stateTimestamps != null ? stateTimestamps : new long[JobStatus.values().length],
		state != null ? state : JobStatus.FINISHED,
		failureCause,
		jsonPlan != null ? jsonPlan : "{\"jobid\":\"" + jobID + "\", \"name\":\"" + jobName + "\", \"nodes\":[]}",
		archivedUserAccumulators != null ? archivedUserAccumulators : new StringifiedAccumulatorResult[0],
		serializedUserAccumulators != null ? serializedUserAccumulators : Collections.emptyMap(),
		archivedExecutionConfig != null ? archivedExecutionConfig : new ArchivedExecutionConfigBuilder().build(),
		isStoppable,
		null,
		null
	);
}
 

private Collection<JobDetails> randomJobDetails(Random rnd) {
	final JobDetails[] details = new JobDetails[rnd.nextInt(10)];
	for (int k = 0; k < details.length; k++) {
		int[] numVerticesPerState = new int[ExecutionState.values().length];
		int numTotal = 0;

		for (int i = 0; i < numVerticesPerState.length; i++) {
			int count = rnd.nextInt(55);
			numVerticesPerState[i] = count;
			numTotal += count;
		}

		long time = rnd.nextLong();
		long endTime = rnd.nextBoolean() ? -1L : time + rnd.nextInt();
		long lastModified = endTime == -1 ? time + rnd.nextInt() : endTime;

		String name = new GenericMessageTester.StringInstantiator().instantiate(rnd);
		JobID jid = new JobID();
		JobStatus status = JobStatus.values()[rnd.nextInt(JobStatus.values().length)];

		details[k] = new JobDetails(jid, name, time, endTime, endTime - time, status, lastModified, numVerticesPerState, numTotal);
	}
	return Arrays.asList(details);
}
 

public ArchivedExecutionGraph build() {
	JobID jobID = this.jobID != null ? this.jobID : new JobID();
	String jobName = this.jobName != null ? this.jobName : "job_" + RANDOM.nextInt();

	if (tasks == null) {
		tasks = Collections.emptyMap();
	}

	return new ArchivedExecutionGraph(
		jobID,
		jobName,
		tasks,
		verticesInCreationOrder != null ? verticesInCreationOrder : new ArrayList<>(tasks.values()),
		stateTimestamps != null ? stateTimestamps : new long[JobStatus.values().length],
		state != null ? state : JobStatus.FINISHED,
		failureCause,
		jsonPlan != null ? jsonPlan : "{\"jobid\":\"" + jobID + "\", \"name\":\"" + jobName + "\", \"nodes\":[]}",
		archivedUserAccumulators != null ? archivedUserAccumulators : new StringifiedAccumulatorResult[0],
		serializedUserAccumulators != null ? serializedUserAccumulators : Collections.emptyMap(),
		archivedExecutionConfig != null ? archivedExecutionConfig : new ArchivedExecutionConfigBuilder().build(),
		isStoppable,
		null,
		null
	);
}
 

/**
 * Create a {@link ArchivedExecutionGraph} from the given {@link ExecutionGraph}.
 *
 * @param executionGraph to create the ArchivedExecutionGraph from
 * @return ArchivedExecutionGraph created from the given ExecutionGraph
 */
public static ArchivedExecutionGraph createFrom(ExecutionGraph executionGraph) {
	final int numberVertices = executionGraph.getTotalNumberOfVertices();

	Map<JobVertexID, ArchivedExecutionJobVertex> archivedTasks = new HashMap<>(numberVertices);
	List<ArchivedExecutionJobVertex> archivedVerticesInCreationOrder = new ArrayList<>(numberVertices);

	for (ExecutionJobVertex task : executionGraph.getVerticesTopologically()) {
		ArchivedExecutionJobVertex archivedTask = task.archive();
		archivedVerticesInCreationOrder.add(archivedTask);
		archivedTasks.put(task.getJobVertexId(), archivedTask);
	}

	final Map<String, SerializedValue<OptionalFailure<Object>>> serializedUserAccumulators =
		executionGraph.getAccumulatorsSerialized();

	final long[] timestamps = new long[JobStatus.values().length];

	for (JobStatus jobStatus : JobStatus.values()) {
		final int ordinal = jobStatus.ordinal();
		timestamps[ordinal] = executionGraph.getStatusTimestamp(jobStatus);
	}

	return new ArchivedExecutionGraph(
		executionGraph.getJobID(),
		executionGraph.getJobName(),
		archivedTasks,
		archivedVerticesInCreationOrder,
		timestamps,
		executionGraph.getState(),
		executionGraph.getFailureInfo(),
		executionGraph.getJsonPlan(),
		executionGraph.getAccumulatorResultsStringified(),
		serializedUserAccumulators,
		executionGraph.getArchivedExecutionConfig(),
		executionGraph.isStoppable(),
		executionGraph.getCheckpointCoordinatorConfiguration(),
		executionGraph.getCheckpointStatsSnapshot());
}
 

@Override
protected JobDetailsInfo getTestResponseInstance() throws Exception {
	final Random random = new Random();
	final int numJobVertexDetailsInfos = 4;
	final String jsonPlan = "{\"id\":\"1234\"}";

	final Map<JobStatus, Long> timestamps = new HashMap<>(JobStatus.values().length);
	final Collection<JobDetailsInfo.JobVertexDetailsInfo> jobVertexInfos = new ArrayList<>(numJobVertexDetailsInfos);
	final Map<ExecutionState, Integer> jobVerticesPerState = new HashMap<>(ExecutionState.values().length);

	for (JobStatus jobStatus : JobStatus.values()) {
		timestamps.put(jobStatus, random.nextLong());
	}

	for (int i = 0; i < numJobVertexDetailsInfos; i++) {
		jobVertexInfos.add(createJobVertexDetailsInfo(random));
	}

	for (ExecutionState executionState : ExecutionState.values()) {
		jobVerticesPerState.put(executionState, random.nextInt());
	}

	return new JobDetailsInfo(
		new JobID(),
		"foobar",
		true,
		JobStatus.values()[random.nextInt(JobStatus.values().length)],
		1L,
		2L,
		1L,
		1984L,
		timestamps,
		jobVertexInfos,
		jobVerticesPerState,
		jsonPlan);
}
 

/**
 * Create a {@link ArchivedExecutionGraph} from the given {@link ExecutionGraph}.
 *
 * @param executionGraph to create the ArchivedExecutionGraph from
 * @return ArchivedExecutionGraph created from the given ExecutionGraph
 */
public static ArchivedExecutionGraph createFrom(ExecutionGraph executionGraph) {
	final int numberVertices = executionGraph.getTotalNumberOfVertices();

	Map<JobVertexID, ArchivedExecutionJobVertex> archivedTasks = new HashMap<>(numberVertices);
	List<ArchivedExecutionJobVertex> archivedVerticesInCreationOrder = new ArrayList<>(numberVertices);

	for (ExecutionJobVertex task : executionGraph.getVerticesTopologically()) {
		ArchivedExecutionJobVertex archivedTask = task.archive();
		archivedVerticesInCreationOrder.add(archivedTask);
		archivedTasks.put(task.getJobVertexId(), archivedTask);
	}

	final Map<String, SerializedValue<OptionalFailure<Object>>> serializedUserAccumulators =
		executionGraph.getAccumulatorsSerialized();

	final long[] timestamps = new long[JobStatus.values().length];

	for (JobStatus jobStatus : JobStatus.values()) {
		final int ordinal = jobStatus.ordinal();
		timestamps[ordinal] = executionGraph.getStatusTimestamp(jobStatus);
	}

	return new ArchivedExecutionGraph(
		executionGraph.getJobID(),
		executionGraph.getJobName(),
		archivedTasks,
		archivedVerticesInCreationOrder,
		timestamps,
		executionGraph.getState(),
		executionGraph.getFailureInfo(),
		executionGraph.getJsonPlan(),
		executionGraph.getAccumulatorResultsStringified(),
		serializedUserAccumulators,
		executionGraph.getArchivedExecutionConfig(),
		executionGraph.isStoppable(),
		executionGraph.getCheckpointCoordinatorConfiguration(),
		executionGraph.getCheckpointStatsSnapshot());
}
 

@Override
protected JobDetailsInfo getTestResponseInstance() throws Exception {
	final Random random = new Random();
	final int numJobVertexDetailsInfos = 4;
	final String jsonPlan = "{\"id\":\"1234\"}";

	final Map<JobStatus, Long> timestamps = new HashMap<>(JobStatus.values().length);
	final Collection<JobDetailsInfo.JobVertexDetailsInfo> jobVertexInfos = new ArrayList<>(numJobVertexDetailsInfos);
	final Map<ExecutionState, Integer> jobVerticesPerState = new HashMap<>(ExecutionState.values().length);

	for (JobStatus jobStatus : JobStatus.values()) {
		timestamps.put(jobStatus, random.nextLong());
	}

	for (int i = 0; i < numJobVertexDetailsInfos; i++) {
		jobVertexInfos.add(createJobVertexDetailsInfo(random));
	}

	for (ExecutionState executionState : ExecutionState.values()) {
		jobVerticesPerState.put(executionState, random.nextInt());
	}

	return new JobDetailsInfo(
		new JobID(),
		"foobar",
		true,
		JobStatus.values()[random.nextInt(JobStatus.values().length)],
		1L,
		2L,
		1L,
		1984L,
		timestamps,
		jobVertexInfos,
		jobVerticesPerState,
		jsonPlan);
}
 

private static JobDetailsInfo createJobDetailsInfo(AccessExecutionGraph executionGraph, @Nullable MetricFetcher metricFetcher) {
	final long now = System.currentTimeMillis();
	final long startTime = executionGraph.getStatusTimestamp(JobStatus.CREATED);
	final long endTime = executionGraph.getState().isGloballyTerminalState() ?
		executionGraph.getStatusTimestamp(executionGraph.getState()) : -1L;
	final long duration = (endTime > 0L ? endTime : now) - startTime;

	final Map<JobStatus, Long> timestamps = new HashMap<>(JobStatus.values().length);

	for (JobStatus jobStatus : JobStatus.values()) {
		timestamps.put(jobStatus, executionGraph.getStatusTimestamp(jobStatus));
	}

	Collection<JobDetailsInfo.JobVertexDetailsInfo> jobVertexInfos = new ArrayList<>(executionGraph.getAllVertices().size());
	int[] jobVerticesPerState = new int[ExecutionState.values().length];

	for (AccessExecutionJobVertex accessExecutionJobVertex : executionGraph.getVerticesTopologically()) {
		final JobDetailsInfo.JobVertexDetailsInfo vertexDetailsInfo = createJobVertexDetailsInfo(
			accessExecutionJobVertex,
			now,
			executionGraph.getJobID(),
			metricFetcher);

		jobVertexInfos.add(vertexDetailsInfo);
		jobVerticesPerState[vertexDetailsInfo.getExecutionState().ordinal()]++;
	}

	Map<ExecutionState, Integer> jobVerticesPerStateMap = new HashMap<>(ExecutionState.values().length);

	for (ExecutionState executionState : ExecutionState.values()) {
		jobVerticesPerStateMap.put(executionState, jobVerticesPerState[executionState.ordinal()]);
	}

	return new JobDetailsInfo(
		executionGraph.getJobID(),
		executionGraph.getJobName(),
		executionGraph.isStoppable(),
		executionGraph.getState(),
		startTime,
		endTime,
		duration,
		now,
		timestamps,
		jobVertexInfos,
		jobVerticesPerStateMap,
		executionGraph.getJsonPlan());
}
 

public ExecutionGraph(
		JobInformation jobInformation,
		ScheduledExecutorService futureExecutor,
		Executor ioExecutor,
		Time rpcTimeout,
		RestartStrategy restartStrategy,
		FailoverStrategy.Factory failoverStrategyFactory,
		SlotProvider slotProvider,
		ClassLoader userClassLoader,
		BlobWriter blobWriter,
		Time allocationTimeout) throws IOException {

	checkNotNull(futureExecutor);

	this.jobInformation = Preconditions.checkNotNull(jobInformation);

	this.blobWriter = Preconditions.checkNotNull(blobWriter);

	this.jobInformationOrBlobKey = BlobWriter.serializeAndTryOffload(jobInformation, jobInformation.getJobId(), blobWriter);

	this.futureExecutor = Preconditions.checkNotNull(futureExecutor);
	this.ioExecutor = Preconditions.checkNotNull(ioExecutor);

	this.slotProvider = Preconditions.checkNotNull(slotProvider, "scheduler");
	this.userClassLoader = Preconditions.checkNotNull(userClassLoader, "userClassLoader");

	this.tasks = new ConcurrentHashMap<>(16);
	this.intermediateResults = new ConcurrentHashMap<>(16);
	this.verticesInCreationOrder = new ArrayList<>(16);
	this.currentExecutions = new ConcurrentHashMap<>(16);

	this.jobStatusListeners  = new CopyOnWriteArrayList<>();
	this.executionListeners = new CopyOnWriteArrayList<>();

	this.stateTimestamps = new long[JobStatus.values().length];
	this.stateTimestamps[JobStatus.CREATED.ordinal()] = System.currentTimeMillis();

	this.rpcTimeout = checkNotNull(rpcTimeout);
	this.allocationTimeout = checkNotNull(allocationTimeout);

	this.restartStrategy = restartStrategy;
	this.kvStateLocationRegistry = new KvStateLocationRegistry(jobInformation.getJobId(), getAllVertices());

	this.verticesFinished = new AtomicInteger();

	this.globalModVersion = 1L;

	// the failover strategy must be instantiated last, so that the execution graph
	// is ready by the time the failover strategy sees it
	this.failoverStrategy = checkNotNull(failoverStrategyFactory.create(this), "null failover strategy");

	this.schedulingFuture = null;
	this.jobMasterMainThreadExecutor =
		new ComponentMainThreadExecutor.DummyComponentMainThreadExecutor(
			"ExecutionGraph is not initialized with proper main thread executor. " +
				"Call to ExecutionGraph.start(...) required.");

	LOG.info("Job recovers via failover strategy: {}", failoverStrategy.getStrategyName());
}
 

public static JobStatus randomJobStatus(Random rnd) {
	return JobStatus.values()[rnd.nextInt(JobStatus.values().length)];
}
 

private static JobDetailsInfo createJobDetailsInfo(AccessExecutionGraph executionGraph, @Nullable MetricFetcher metricFetcher) {
	final long now = System.currentTimeMillis();
	final long startTime = executionGraph.getStatusTimestamp(JobStatus.CREATED);
	final long endTime = executionGraph.getState().isGloballyTerminalState() ?
		executionGraph.getStatusTimestamp(executionGraph.getState()) : -1L;
	final long duration = (endTime > 0L ? endTime : now) - startTime;

	final Map<JobStatus, Long> timestamps = new HashMap<>(JobStatus.values().length);

	for (JobStatus jobStatus : JobStatus.values()) {
		timestamps.put(jobStatus, executionGraph.getStatusTimestamp(jobStatus));
	}

	Collection<JobDetailsInfo.JobVertexDetailsInfo> jobVertexInfos = new ArrayList<>(executionGraph.getAllVertices().size());
	int[] jobVerticesPerState = new int[ExecutionState.values().length];

	for (AccessExecutionJobVertex accessExecutionJobVertex : executionGraph.getVerticesTopologically()) {
		final JobDetailsInfo.JobVertexDetailsInfo vertexDetailsInfo = createJobVertexDetailsInfo(
			accessExecutionJobVertex,
			now,
			executionGraph.getJobID(),
			metricFetcher);

		jobVertexInfos.add(vertexDetailsInfo);
		jobVerticesPerState[vertexDetailsInfo.getExecutionState().ordinal()]++;
	}

	Map<ExecutionState, Integer> jobVerticesPerStateMap = new HashMap<>(ExecutionState.values().length);

	for (ExecutionState executionState : ExecutionState.values()) {
		jobVerticesPerStateMap.put(executionState, jobVerticesPerState[executionState.ordinal()]);
	}

	return new JobDetailsInfo(
		executionGraph.getJobID(),
		executionGraph.getJobName(),
		executionGraph.isStoppable(),
		executionGraph.getState(),
		startTime,
		endTime,
		duration,
		now,
		timestamps,
		jobVertexInfos,
		jobVerticesPerStateMap,
		executionGraph.getJsonPlan());
}
 

public ExecutionGraph(
		JobInformation jobInformation,
		ScheduledExecutorService futureExecutor,
		Executor ioExecutor,
		Time rpcTimeout,
		RestartStrategy restartStrategy,
		int maxPriorAttemptsHistoryLength,
		FailoverStrategy.Factory failoverStrategyFactory,
		SlotProvider slotProvider,
		ClassLoader userClassLoader,
		BlobWriter blobWriter,
		Time allocationTimeout,
		PartitionReleaseStrategy.Factory partitionReleaseStrategyFactory,
		ShuffleMaster<?> shuffleMaster,
		PartitionTracker partitionTracker,
		ScheduleMode scheduleMode,
		boolean allowQueuedScheduling) throws IOException {

	checkNotNull(futureExecutor);

	this.jobInformation = Preconditions.checkNotNull(jobInformation);

	this.blobWriter = Preconditions.checkNotNull(blobWriter);

	this.scheduleMode = checkNotNull(scheduleMode);

	this.allowQueuedScheduling = allowQueuedScheduling;

	this.jobInformationOrBlobKey = BlobWriter.serializeAndTryOffload(jobInformation, jobInformation.getJobId(), blobWriter);

	this.futureExecutor = Preconditions.checkNotNull(futureExecutor);
	this.ioExecutor = Preconditions.checkNotNull(ioExecutor);

	this.slotProviderStrategy = SlotProviderStrategy.from(
		scheduleMode,
		slotProvider,
		allocationTimeout,
		allowQueuedScheduling);
	this.userClassLoader = Preconditions.checkNotNull(userClassLoader, "userClassLoader");

	this.tasks = new ConcurrentHashMap<>(16);
	this.intermediateResults = new ConcurrentHashMap<>(16);
	this.verticesInCreationOrder = new ArrayList<>(16);
	this.currentExecutions = new ConcurrentHashMap<>(16);

	this.jobStatusListeners  = new CopyOnWriteArrayList<>();

	this.stateTimestamps = new long[JobStatus.values().length];
	this.stateTimestamps[JobStatus.CREATED.ordinal()] = System.currentTimeMillis();

	this.rpcTimeout = checkNotNull(rpcTimeout);
	this.allocationTimeout = checkNotNull(allocationTimeout);

	this.partitionReleaseStrategyFactory = checkNotNull(partitionReleaseStrategyFactory);

	this.restartStrategy = restartStrategy;
	this.kvStateLocationRegistry = new KvStateLocationRegistry(jobInformation.getJobId(), getAllVertices());

	this.verticesFinished = new AtomicInteger();

	this.globalModVersion = 1L;

	// the failover strategy must be instantiated last, so that the execution graph
	// is ready by the time the failover strategy sees it
	this.failoverStrategy = checkNotNull(failoverStrategyFactory.create(this), "null failover strategy");

	this.maxPriorAttemptsHistoryLength = maxPriorAttemptsHistoryLength;

	this.schedulingFuture = null;
	this.jobMasterMainThreadExecutor =
		new ComponentMainThreadExecutor.DummyComponentMainThreadExecutor(
			"ExecutionGraph is not initialized with proper main thread executor. " +
				"Call to ExecutionGraph.start(...) required.");

	this.shuffleMaster = checkNotNull(shuffleMaster);

	this.partitionTracker = checkNotNull(partitionTracker);

	this.resultPartitionAvailabilityChecker = new ExecutionGraphResultPartitionAvailabilityChecker(
		this::createResultPartitionId,
		partitionTracker);

	LOG.info("Job recovers via failover strategy: {}", failoverStrategy.getStrategyName());
}
 

public static JobStatus randomJobStatus(Random rnd) {
	return JobStatus.values()[rnd.nextInt(JobStatus.values().length)];
}