org.apache.flink.api.common.ArchivedExecutionConfig Java Examples

The following examples show how to use org.apache.flink.api.common.ArchivedExecutionConfig. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: JobConfigHandler.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
private static JobConfigInfo createJobConfigInfo(AccessExecutionGraph executionGraph) {
	final ArchivedExecutionConfig executionConfig = executionGraph.getArchivedExecutionConfig();
	final JobConfigInfo.ExecutionConfigInfo executionConfigInfo;

	if (executionConfig != null) {
		executionConfigInfo = new JobConfigInfo.ExecutionConfigInfo(
			executionConfig.getExecutionMode(),
			executionConfig.getRestartStrategyDescription(),
			executionConfig.getParallelism(),
			executionConfig.getObjectReuseEnabled(),
			executionConfig.getGlobalJobParameters());
	} else {
		executionConfigInfo = null;
	}

	return new JobConfigInfo(executionGraph.getJobID(), executionGraph.getJobName(), executionConfigInfo);
}
Example #2
Source File: DefaultExecutionGraphCacheTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
public SuspendableAccessExecutionGraph(JobID jobId) {
	super(
		jobId,
		"DefaultExecutionGraphCacheTest",
		Collections.emptyMap(),
		Collections.emptyList(),
		new long[0],
		JobStatus.RUNNING,
		new ErrorInfo(new FlinkException("Test"), 42L),
		"",
		new StringifiedAccumulatorResult[0],
		Collections.emptyMap(),
		new ArchivedExecutionConfig(new ExecutionConfig()),
		false,
		null,
		null,
		"stateBackendName");

	jobStatus = super.getState();
}
Example #3
Source File: ExecutionGraphCacheTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
public SuspendableAccessExecutionGraph(JobID jobId) {
	super(
		jobId,
		"ExecutionGraphCacheTest",
		Collections.emptyMap(),
		Collections.emptyList(),
		new long[0],
		JobStatus.RUNNING,
		new ErrorInfo(new FlinkException("Test"), 42L),
		"",
		new StringifiedAccumulatorResult[0],
		Collections.emptyMap(),
		new ArchivedExecutionConfig(new ExecutionConfig()),
		false,
		null,
		null);

	jobStatus = super.getState();
}
Example #4
Source File: JobConfigHandler.java    From flink with Apache License 2.0 6 votes vote down vote up 
private static JobConfigInfo createJobConfigInfo(AccessExecutionGraph executionGraph) {
	final ArchivedExecutionConfig executionConfig = executionGraph.getArchivedExecutionConfig();
	final JobConfigInfo.ExecutionConfigInfo executionConfigInfo;

	if (executionConfig != null) {
		executionConfigInfo = new JobConfigInfo.ExecutionConfigInfo(
			executionConfig.getExecutionMode(),
			executionConfig.getRestartStrategyDescription(),
			executionConfig.getParallelism(),
			executionConfig.getObjectReuseEnabled(),
			executionConfig.getGlobalJobParameters());
	} else {
		executionConfigInfo = null;
	}

	return new JobConfigInfo(executionGraph.getJobID(), executionGraph.getJobName(), executionConfigInfo);
}
Example #5
Source File: ExecutionGraphCacheTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
public SuspendableAccessExecutionGraph(JobID jobId) {
	super(
		jobId,
		"ExecutionGraphCacheTest",
		Collections.emptyMap(),
		Collections.emptyList(),
		new long[0],
		JobStatus.RUNNING,
		new ErrorInfo(new FlinkException("Test"), 42L),
		"",
		new StringifiedAccumulatorResult[0],
		Collections.emptyMap(),
		new ArchivedExecutionConfig(new ExecutionConfig()),
		false,
		null,
		null);

	jobStatus = super.getState();
}
Example #6
Source File: ExecutionGraph.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
/**
 * Returns the serializable {@link ArchivedExecutionConfig}.
 *
 * @return ArchivedExecutionConfig which may be null in case of errors
 */
@Override
public ArchivedExecutionConfig getArchivedExecutionConfig() {
	// create a summary of all relevant data accessed in the web interface's JobConfigHandler
	try {
		ExecutionConfig executionConfig = jobInformation.getSerializedExecutionConfig().deserializeValue(userClassLoader);
		if (executionConfig != null) {
			return executionConfig.archive();
		}
	} catch (IOException | ClassNotFoundException e) {
		LOG.error("Couldn't create ArchivedExecutionConfig for job {} ", getJobID(), e);
	}
	return null;
}
Example #7
Source File: ArchivedExecutionConfigBuilder.java    From flink with Apache License 2.0 5 votes vote down vote up 
public ArchivedExecutionConfig build() {
	return new ArchivedExecutionConfig(
		executionMode != null ? executionMode : ExecutionMode.PIPELINED.name(),
		restartStrategyDescription != null ? restartStrategyDescription : "default",
		parallelism,
		objectReuseEnabled,
		globalJobParameters != null ? globalJobParameters : Collections.<String, String>emptyMap()
	);
}
Example #8
Source File: JobConfigHandlerTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void handleRequest_executionConfigWithSecretValues_excludesSecretValuesFromResponse() throws HandlerRequestException {
	final JobConfigHandler jobConfigHandler = new JobConfigHandler(
		() -> null,
		TestingUtils.TIMEOUT(),
		Collections.emptyMap(),
		JobConfigHeaders.getInstance(),
		new DefaultExecutionGraphCache(TestingUtils.TIMEOUT(), TestingUtils.TIMEOUT()),
		TestingUtils.defaultExecutor());

	final Map<String, String> globalJobParameters = new HashMap<>();
	globalJobParameters.put("foobar", "barfoo");
	globalJobParameters.put("bar.secret.foo", "my secret");
	globalJobParameters.put("password.to.my.safe", "12345");

	final ArchivedExecutionConfig archivedExecutionConfig = new ArchivedExecutionConfigBuilder()
		.setGlobalJobParameters(globalJobParameters)
		.build();
	final AccessExecutionGraph archivedExecutionGraph = new ArchivedExecutionGraphBuilder()
		.setArchivedExecutionConfig(archivedExecutionConfig)
		.build();
	final HandlerRequest<EmptyRequestBody, JobMessageParameters> handlerRequest = createRequest(archivedExecutionGraph.getJobID());

	final JobConfigInfo jobConfigInfoResponse = jobConfigHandler.handleRequest(handlerRequest, archivedExecutionGraph);

	final Map<String, String> filteredGlobalJobParameters = filterSecretValues(globalJobParameters);

	assertThat(jobConfigInfoResponse.getExecutionConfigInfo().getGlobalJobParameters(), is(equalTo(filteredGlobalJobParameters)));
}
Example #9
Source File: ArchivedExecutionGraph.java    From flink with Apache License 2.0 5 votes vote down vote up 
public ArchivedExecutionGraph(
		JobID jobID,
		String jobName,
		Map<JobVertexID, ArchivedExecutionJobVertex> tasks,
		List<ArchivedExecutionJobVertex> verticesInCreationOrder,
		long[] stateTimestamps,
		JobStatus state,
		@Nullable ErrorInfo failureCause,
		String jsonPlan,
		StringifiedAccumulatorResult[] archivedUserAccumulators,
		Map<String, SerializedValue<OptionalFailure<Object>>> serializedUserAccumulators,
		ArchivedExecutionConfig executionConfig,
		boolean isStoppable,
		@Nullable CheckpointCoordinatorConfiguration jobCheckpointingConfiguration,
		@Nullable CheckpointStatsSnapshot checkpointStatsSnapshot,
		@Nullable String stateBackendName) {

	this.jobID = Preconditions.checkNotNull(jobID);
	this.jobName = Preconditions.checkNotNull(jobName);
	this.tasks = Preconditions.checkNotNull(tasks);
	this.verticesInCreationOrder = Preconditions.checkNotNull(verticesInCreationOrder);
	this.stateTimestamps = Preconditions.checkNotNull(stateTimestamps);
	this.state = Preconditions.checkNotNull(state);
	this.failureCause = failureCause;
	this.jsonPlan = Preconditions.checkNotNull(jsonPlan);
	this.archivedUserAccumulators = Preconditions.checkNotNull(archivedUserAccumulators);
	this.serializedUserAccumulators = Preconditions.checkNotNull(serializedUserAccumulators);
	this.archivedExecutionConfig = Preconditions.checkNotNull(executionConfig);
	this.isStoppable = isStoppable;
	this.jobCheckpointingConfiguration = jobCheckpointingConfiguration;
	this.checkpointStatsSnapshot = checkpointStatsSnapshot;
	this.stateBackendName = stateBackendName;
}
Example #10
Source File: ExecutionGraph.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Returns the serializable {@link ArchivedExecutionConfig}.
 *
 * @return ArchivedExecutionConfig which may be null in case of errors
 */
@Override
public ArchivedExecutionConfig getArchivedExecutionConfig() {
	// create a summary of all relevant data accessed in the web interface's JobConfigHandler
	try {
		ExecutionConfig executionConfig = jobInformation.getSerializedExecutionConfig().deserializeValue(userClassLoader);
		if (executionConfig != null) {
			return executionConfig.archive();
		}
	} catch (IOException | ClassNotFoundException e) {
		LOG.error("Couldn't create ArchivedExecutionConfig for job {} ", getJobID(), e);
	}
	return null;
}
Example #11
Source File: JobConfigHandler.java    From flink with Apache License 2.0 5 votes vote down vote up 
private static JobConfigInfo createJobConfigInfo(AccessExecutionGraph executionGraph) {
	final ArchivedExecutionConfig executionConfig = executionGraph.getArchivedExecutionConfig();
	final JobConfigInfo.ExecutionConfigInfo executionConfigInfo;

	if (executionConfig != null) {
		executionConfigInfo = JobConfigInfo.ExecutionConfigInfo.from(executionConfig);
	} else {
		executionConfigInfo = null;
	}

	return new JobConfigInfo(executionGraph.getJobID(), executionGraph.getJobName(), executionConfigInfo);
}
Example #12
Source File: JobConfigInfo.java    From flink with Apache License 2.0 5 votes vote down vote up 
public static ExecutionConfigInfo from(ArchivedExecutionConfig archivedExecutionConfig) {
	return new ExecutionConfigInfo(
		archivedExecutionConfig.getExecutionMode(),
		archivedExecutionConfig.getRestartStrategyDescription(),
		archivedExecutionConfig.getParallelism(),
		archivedExecutionConfig.getObjectReuseEnabled(),
		ConfigurationUtils.hideSensitiveValues(archivedExecutionConfig.getGlobalJobParameters()));
}
Example #13
Source File: ArchivedExecutionConfigBuilder.java    From flink with Apache License 2.0 5 votes vote down vote up 
public ArchivedExecutionConfig build() {
	return new ArchivedExecutionConfig(
		executionMode != null ? executionMode : ExecutionMode.PIPELINED.name(),
		restartStrategyDescription != null ? restartStrategyDescription : "default",
		parallelism,
		objectReuseEnabled,
		globalJobParameters != null ? globalJobParameters : Collections.<String, String>emptyMap()
	);
}
Example #14
Source File: ArchivedExecutionGraph.java    From flink with Apache License 2.0 5 votes vote down vote up 
public ArchivedExecutionGraph(
		JobID jobID,
		String jobName,
		Map<JobVertexID, ArchivedExecutionJobVertex> tasks,
		List<ArchivedExecutionJobVertex> verticesInCreationOrder,
		long[] stateTimestamps,
		JobStatus state,
		@Nullable ErrorInfo failureCause,
		String jsonPlan,
		StringifiedAccumulatorResult[] archivedUserAccumulators,
		Map<String, SerializedValue<OptionalFailure<Object>>> serializedUserAccumulators,
		ArchivedExecutionConfig executionConfig,
		boolean isStoppable,
		@Nullable CheckpointCoordinatorConfiguration jobCheckpointingConfiguration,
		@Nullable CheckpointStatsSnapshot checkpointStatsSnapshot) {

	this.jobID = Preconditions.checkNotNull(jobID);
	this.jobName = Preconditions.checkNotNull(jobName);
	this.tasks = Preconditions.checkNotNull(tasks);
	this.verticesInCreationOrder = Preconditions.checkNotNull(verticesInCreationOrder);
	this.stateTimestamps = Preconditions.checkNotNull(stateTimestamps);
	this.state = Preconditions.checkNotNull(state);
	this.failureCause = failureCause;
	this.jsonPlan = Preconditions.checkNotNull(jsonPlan);
	this.archivedUserAccumulators = Preconditions.checkNotNull(archivedUserAccumulators);
	this.serializedUserAccumulators = Preconditions.checkNotNull(serializedUserAccumulators);
	this.archivedExecutionConfig = Preconditions.checkNotNull(executionConfig);
	this.isStoppable = isStoppable;
	this.jobCheckpointingConfiguration = jobCheckpointingConfiguration;
	this.checkpointStatsSnapshot = checkpointStatsSnapshot;
}
Example #15
Source File: ExecutionGraph.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Returns the serializable {@link ArchivedExecutionConfig}.
 *
 * @return ArchivedExecutionConfig which may be null in case of errors
 */
@Override
public ArchivedExecutionConfig getArchivedExecutionConfig() {
	// create a summary of all relevant data accessed in the web interface's JobConfigHandler
	try {
		ExecutionConfig executionConfig = jobInformation.getSerializedExecutionConfig().deserializeValue(userClassLoader);
		if (executionConfig != null) {
			return executionConfig.archive();
		}
	} catch (IOException | ClassNotFoundException e) {
		LOG.error("Couldn't create ArchivedExecutionConfig for job {} ", getJobID(), e);
	}
	return null;
}
Example #16
Source File: ArchivedExecutionConfigBuilder.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
public ArchivedExecutionConfig build() {
	return new ArchivedExecutionConfig(
		executionMode != null ? executionMode : ExecutionMode.PIPELINED.name(),
		restartStrategyDescription != null ? restartStrategyDescription : "default",
		parallelism,
		objectReuseEnabled,
		globalJobParameters != null ? globalJobParameters : Collections.<String, String>emptyMap()
	);
}
Example #17
Source File: ArchivedExecutionGraph.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
public ArchivedExecutionGraph(
		JobID jobID,
		String jobName,
		Map<JobVertexID, ArchivedExecutionJobVertex> tasks,
		List<ArchivedExecutionJobVertex> verticesInCreationOrder,
		long[] stateTimestamps,
		JobStatus state,
		@Nullable ErrorInfo failureCause,
		String jsonPlan,
		StringifiedAccumulatorResult[] archivedUserAccumulators,
		Map<String, SerializedValue<OptionalFailure<Object>>> serializedUserAccumulators,
		ArchivedExecutionConfig executionConfig,
		boolean isStoppable,
		@Nullable CheckpointCoordinatorConfiguration jobCheckpointingConfiguration,
		@Nullable CheckpointStatsSnapshot checkpointStatsSnapshot) {

	this.jobID = Preconditions.checkNotNull(jobID);
	this.jobName = Preconditions.checkNotNull(jobName);
	this.tasks = Preconditions.checkNotNull(tasks);
	this.verticesInCreationOrder = Preconditions.checkNotNull(verticesInCreationOrder);
	this.stateTimestamps = Preconditions.checkNotNull(stateTimestamps);
	this.state = Preconditions.checkNotNull(state);
	this.failureCause = failureCause;
	this.jsonPlan = Preconditions.checkNotNull(jsonPlan);
	this.archivedUserAccumulators = Preconditions.checkNotNull(archivedUserAccumulators);
	this.serializedUserAccumulators = Preconditions.checkNotNull(serializedUserAccumulators);
	this.archivedExecutionConfig = Preconditions.checkNotNull(executionConfig);
	this.isStoppable = isStoppable;
	this.jobCheckpointingConfiguration = jobCheckpointingConfiguration;
	this.checkpointStatsSnapshot = checkpointStatsSnapshot;
}
Example #18
Source File: ArchivedExecutionGraph.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Override
public ArchivedExecutionConfig getArchivedExecutionConfig() {
	return archivedExecutionConfig;
}
Example #19
Source File: ArchivedExecutionGraphBuilder.java    From flink with Apache License 2.0 4 votes vote down vote up 
public ArchivedExecutionGraphBuilder setArchivedExecutionConfig(ArchivedExecutionConfig archivedExecutionConfig) {
	this.archivedExecutionConfig = archivedExecutionConfig;
	return this;
}
Example #20
Source File: ArchivedExecutionGraphTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
private static void compareExecutionGraph(AccessExecutionGraph runtimeGraph, AccessExecutionGraph archivedGraph) throws IOException, ClassNotFoundException {
	assertTrue(archivedGraph.isArchived());
	// -------------------------------------------------------------------------------------------------------------
	// ExecutionGraph
	// -------------------------------------------------------------------------------------------------------------
	assertEquals(runtimeGraph.getJsonPlan(), archivedGraph.getJsonPlan());
	assertEquals(runtimeGraph.getJobID(), archivedGraph.getJobID());
	assertEquals(runtimeGraph.getJobName(), archivedGraph.getJobName());
	assertEquals(runtimeGraph.getState(), archivedGraph.getState());
	assertEquals(runtimeGraph.getFailureInfo().getExceptionAsString(), archivedGraph.getFailureInfo().getExceptionAsString());
	assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.CREATED), archivedGraph.getStatusTimestamp(JobStatus.CREATED));
	assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.RUNNING), archivedGraph.getStatusTimestamp(JobStatus.RUNNING));
	assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.FAILING), archivedGraph.getStatusTimestamp(JobStatus.FAILING));
	assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.FAILED), archivedGraph.getStatusTimestamp(JobStatus.FAILED));
	assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.CANCELLING), archivedGraph.getStatusTimestamp(JobStatus.CANCELLING));
	assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.CANCELED), archivedGraph.getStatusTimestamp(JobStatus.CANCELED));
	assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.FINISHED), archivedGraph.getStatusTimestamp(JobStatus.FINISHED));
	assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.RESTARTING), archivedGraph.getStatusTimestamp(JobStatus.RESTARTING));
	assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.SUSPENDED), archivedGraph.getStatusTimestamp(JobStatus.SUSPENDED));
	assertEquals(runtimeGraph.isStoppable(), archivedGraph.isStoppable());

	// -------------------------------------------------------------------------------------------------------------
	// CheckpointStats
	// -------------------------------------------------------------------------------------------------------------
	CheckpointStatsSnapshot runtimeSnapshot = runtimeGraph.getCheckpointStatsSnapshot();
	CheckpointStatsSnapshot archivedSnapshot = archivedGraph.getCheckpointStatsSnapshot();

	assertEquals(runtimeSnapshot.getSummaryStats().getEndToEndDurationStats().getAverage(), archivedSnapshot.getSummaryStats().getEndToEndDurationStats().getAverage());
	assertEquals(runtimeSnapshot.getSummaryStats().getEndToEndDurationStats().getMinimum(), archivedSnapshot.getSummaryStats().getEndToEndDurationStats().getMinimum());
	assertEquals(runtimeSnapshot.getSummaryStats().getEndToEndDurationStats().getMaximum(), archivedSnapshot.getSummaryStats().getEndToEndDurationStats().getMaximum());

	assertEquals(runtimeSnapshot.getSummaryStats().getStateSizeStats().getAverage(), archivedSnapshot.getSummaryStats().getStateSizeStats().getAverage());
	assertEquals(runtimeSnapshot.getSummaryStats().getStateSizeStats().getMinimum(), archivedSnapshot.getSummaryStats().getStateSizeStats().getMinimum());
	assertEquals(runtimeSnapshot.getSummaryStats().getStateSizeStats().getMaximum(), archivedSnapshot.getSummaryStats().getStateSizeStats().getMaximum());

	assertEquals(runtimeSnapshot.getCounts().getTotalNumberOfCheckpoints(), archivedSnapshot.getCounts().getTotalNumberOfCheckpoints());
	assertEquals(runtimeSnapshot.getCounts().getNumberOfCompletedCheckpoints(), archivedSnapshot.getCounts().getNumberOfCompletedCheckpoints());
	assertEquals(runtimeSnapshot.getCounts().getNumberOfInProgressCheckpoints(), archivedSnapshot.getCounts().getNumberOfInProgressCheckpoints());

	// -------------------------------------------------------------------------------------------------------------
	// ArchivedExecutionConfig
	// -------------------------------------------------------------------------------------------------------------
	ArchivedExecutionConfig runtimeConfig = runtimeGraph.getArchivedExecutionConfig();
	ArchivedExecutionConfig archivedConfig = archivedGraph.getArchivedExecutionConfig();

	assertEquals(runtimeConfig.getExecutionMode(), archivedConfig.getExecutionMode());
	assertEquals(runtimeConfig.getParallelism(), archivedConfig.getParallelism());
	assertEquals(runtimeConfig.getObjectReuseEnabled(), archivedConfig.getObjectReuseEnabled());
	assertEquals(runtimeConfig.getRestartStrategyDescription(), archivedConfig.getRestartStrategyDescription());
	assertNotNull(archivedConfig.getGlobalJobParameters().get("hello"));
	assertEquals(runtimeConfig.getGlobalJobParameters().get("hello"), archivedConfig.getGlobalJobParameters().get("hello"));

	// -------------------------------------------------------------------------------------------------------------
	// StringifiedAccumulators
	// -------------------------------------------------------------------------------------------------------------
	compareStringifiedAccumulators(runtimeGraph.getAccumulatorResultsStringified(), archivedGraph.getAccumulatorResultsStringified());
	compareSerializedAccumulators(runtimeGraph.getAccumulatorsSerialized(), archivedGraph.getAccumulatorsSerialized());

	// -------------------------------------------------------------------------------------------------------------
	// JobVertices
	// -------------------------------------------------------------------------------------------------------------
	Map<JobVertexID, ? extends AccessExecutionJobVertex> runtimeVertices = runtimeGraph.getAllVertices();
	Map<JobVertexID, ? extends AccessExecutionJobVertex> archivedVertices = archivedGraph.getAllVertices();

	for (Map.Entry<JobVertexID, ? extends AccessExecutionJobVertex> vertex : runtimeVertices.entrySet()) {
		compareExecutionJobVertex(vertex.getValue(), archivedVertices.get(vertex.getKey()));
	}

	Iterator<? extends AccessExecutionJobVertex> runtimeTopologicalVertices = runtimeGraph.getVerticesTopologically().iterator();
	Iterator<? extends AccessExecutionJobVertex> archiveTopologicaldVertices = archivedGraph.getVerticesTopologically().iterator();

	while (runtimeTopologicalVertices.hasNext()) {
		assertTrue(archiveTopologicaldVertices.hasNext());
		compareExecutionJobVertex(runtimeTopologicalVertices.next(), archiveTopologicaldVertices.next());
	}

	// -------------------------------------------------------------------------------------------------------------
	// ExecutionVertices
	// -------------------------------------------------------------------------------------------------------------
	Iterator<? extends AccessExecutionVertex> runtimeExecutionVertices = runtimeGraph.getAllExecutionVertices().iterator();
	Iterator<? extends AccessExecutionVertex> archivedExecutionVertices = archivedGraph.getAllExecutionVertices().iterator();

	while (runtimeExecutionVertices.hasNext()) {
		assertTrue(archivedExecutionVertices.hasNext());
		compareExecutionVertex(runtimeExecutionVertices.next(), archivedExecutionVertices.next());
	}
}
Example #21
Source File: ArchivedExecutionGraphTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
private static void compareExecutionGraph(AccessExecutionGraph runtimeGraph, AccessExecutionGraph archivedGraph) throws IOException, ClassNotFoundException {
	assertTrue(archivedGraph.isArchived());
	// -------------------------------------------------------------------------------------------------------------
	// ExecutionGraph
	// -------------------------------------------------------------------------------------------------------------
	assertEquals(runtimeGraph.getJsonPlan(), archivedGraph.getJsonPlan());
	assertEquals(runtimeGraph.getJobID(), archivedGraph.getJobID());
	assertEquals(runtimeGraph.getJobName(), archivedGraph.getJobName());
	assertEquals(runtimeGraph.getState(), archivedGraph.getState());
	assertEquals(runtimeGraph.getFailureInfo().getExceptionAsString(), archivedGraph.getFailureInfo().getExceptionAsString());
	assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.CREATED), archivedGraph.getStatusTimestamp(JobStatus.CREATED));
	assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.RUNNING), archivedGraph.getStatusTimestamp(JobStatus.RUNNING));
	assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.FAILING), archivedGraph.getStatusTimestamp(JobStatus.FAILING));
	assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.FAILED), archivedGraph.getStatusTimestamp(JobStatus.FAILED));
	assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.CANCELLING), archivedGraph.getStatusTimestamp(JobStatus.CANCELLING));
	assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.CANCELED), archivedGraph.getStatusTimestamp(JobStatus.CANCELED));
	assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.FINISHED), archivedGraph.getStatusTimestamp(JobStatus.FINISHED));
	assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.RESTARTING), archivedGraph.getStatusTimestamp(JobStatus.RESTARTING));
	assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.SUSPENDED), archivedGraph.getStatusTimestamp(JobStatus.SUSPENDED));
	assertEquals(runtimeGraph.isStoppable(), archivedGraph.isStoppable());

	// -------------------------------------------------------------------------------------------------------------
	// CheckpointStats
	// -------------------------------------------------------------------------------------------------------------
	CheckpointStatsSnapshot runtimeSnapshot = runtimeGraph.getCheckpointStatsSnapshot();
	CheckpointStatsSnapshot archivedSnapshot = archivedGraph.getCheckpointStatsSnapshot();

	assertEquals(runtimeSnapshot.getSummaryStats().getEndToEndDurationStats().getAverage(), archivedSnapshot.getSummaryStats().getEndToEndDurationStats().getAverage());
	assertEquals(runtimeSnapshot.getSummaryStats().getEndToEndDurationStats().getMinimum(), archivedSnapshot.getSummaryStats().getEndToEndDurationStats().getMinimum());
	assertEquals(runtimeSnapshot.getSummaryStats().getEndToEndDurationStats().getMaximum(), archivedSnapshot.getSummaryStats().getEndToEndDurationStats().getMaximum());

	assertEquals(runtimeSnapshot.getSummaryStats().getStateSizeStats().getAverage(), archivedSnapshot.getSummaryStats().getStateSizeStats().getAverage());
	assertEquals(runtimeSnapshot.getSummaryStats().getStateSizeStats().getMinimum(), archivedSnapshot.getSummaryStats().getStateSizeStats().getMinimum());
	assertEquals(runtimeSnapshot.getSummaryStats().getStateSizeStats().getMaximum(), archivedSnapshot.getSummaryStats().getStateSizeStats().getMaximum());

	assertEquals(runtimeSnapshot.getCounts().getTotalNumberOfCheckpoints(), archivedSnapshot.getCounts().getTotalNumberOfCheckpoints());
	assertEquals(runtimeSnapshot.getCounts().getNumberOfCompletedCheckpoints(), archivedSnapshot.getCounts().getNumberOfCompletedCheckpoints());
	assertEquals(runtimeSnapshot.getCounts().getNumberOfInProgressCheckpoints(), archivedSnapshot.getCounts().getNumberOfInProgressCheckpoints());

	// -------------------------------------------------------------------------------------------------------------
	// ArchivedExecutionConfig
	// -------------------------------------------------------------------------------------------------------------
	ArchivedExecutionConfig runtimeConfig = runtimeGraph.getArchivedExecutionConfig();
	ArchivedExecutionConfig archivedConfig = archivedGraph.getArchivedExecutionConfig();

	assertEquals(runtimeConfig.getExecutionMode(), archivedConfig.getExecutionMode());
	assertEquals(runtimeConfig.getParallelism(), archivedConfig.getParallelism());
	assertEquals(runtimeConfig.getObjectReuseEnabled(), archivedConfig.getObjectReuseEnabled());
	assertEquals(runtimeConfig.getRestartStrategyDescription(), archivedConfig.getRestartStrategyDescription());
	assertNotNull(archivedConfig.getGlobalJobParameters().get("hello"));
	assertEquals(runtimeConfig.getGlobalJobParameters().get("hello"), archivedConfig.getGlobalJobParameters().get("hello"));

	// -------------------------------------------------------------------------------------------------------------
	// StringifiedAccumulators
	// -------------------------------------------------------------------------------------------------------------
	compareStringifiedAccumulators(runtimeGraph.getAccumulatorResultsStringified(), archivedGraph.getAccumulatorResultsStringified());
	compareSerializedAccumulators(runtimeGraph.getAccumulatorsSerialized(), archivedGraph.getAccumulatorsSerialized());

	// -------------------------------------------------------------------------------------------------------------
	// JobVertices
	// -------------------------------------------------------------------------------------------------------------
	Map<JobVertexID, ? extends AccessExecutionJobVertex> runtimeVertices = runtimeGraph.getAllVertices();
	Map<JobVertexID, ? extends AccessExecutionJobVertex> archivedVertices = archivedGraph.getAllVertices();

	for (Map.Entry<JobVertexID, ? extends AccessExecutionJobVertex> vertex : runtimeVertices.entrySet()) {
		compareExecutionJobVertex(vertex.getValue(), archivedVertices.get(vertex.getKey()));
	}

	Iterator<? extends AccessExecutionJobVertex> runtimeTopologicalVertices = runtimeGraph.getVerticesTopologically().iterator();
	Iterator<? extends AccessExecutionJobVertex> archiveTopologicaldVertices = archivedGraph.getVerticesTopologically().iterator();

	while (runtimeTopologicalVertices.hasNext()) {
		assertTrue(archiveTopologicaldVertices.hasNext());
		compareExecutionJobVertex(runtimeTopologicalVertices.next(), archiveTopologicaldVertices.next());
	}

	// -------------------------------------------------------------------------------------------------------------
	// ExecutionVertices
	// -------------------------------------------------------------------------------------------------------------
	Iterator<? extends AccessExecutionVertex> runtimeExecutionVertices = runtimeGraph.getAllExecutionVertices().iterator();
	Iterator<? extends AccessExecutionVertex> archivedExecutionVertices = archivedGraph.getAllExecutionVertices().iterator();

	while (runtimeExecutionVertices.hasNext()) {
		assertTrue(archivedExecutionVertices.hasNext());
		compareExecutionVertex(runtimeExecutionVertices.next(), archivedExecutionVertices.next());
	}
}
Example #22
Source File: ArchivedExecutionGraphBuilder.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
public ArchivedExecutionGraphBuilder setArchivedExecutionConfig(ArchivedExecutionConfig archivedExecutionConfig) {
	this.archivedExecutionConfig = archivedExecutionConfig;
	return this;
}
Example #23
Source File: ArchivedExecutionGraph.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Override
public ArchivedExecutionConfig getArchivedExecutionConfig() {
	return archivedExecutionConfig;
}
Example #24
Source File: ArchivedExecutionGraph.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@Override
public ArchivedExecutionConfig getArchivedExecutionConfig() {
	return archivedExecutionConfig;
}
Example #25
Source File: ArchivedExecutionGraphBuilder.java    From flink with Apache License 2.0 4 votes vote down vote up 
public ArchivedExecutionGraphBuilder setArchivedExecutionConfig(ArchivedExecutionConfig archivedExecutionConfig) {
	this.archivedExecutionConfig = archivedExecutionConfig;
	return this;
}
Example #26
Source File: ArchivedExecutionGraphTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
private static void compareExecutionGraph(AccessExecutionGraph runtimeGraph, AccessExecutionGraph archivedGraph) throws IOException, ClassNotFoundException {
	assertTrue(archivedGraph.isArchived());
	// -------------------------------------------------------------------------------------------------------------
	// ExecutionGraph
	// -------------------------------------------------------------------------------------------------------------
	assertEquals(runtimeGraph.getJsonPlan(), archivedGraph.getJsonPlan());
	assertEquals(runtimeGraph.getJobID(), archivedGraph.getJobID());
	assertEquals(runtimeGraph.getJobName(), archivedGraph.getJobName());
	assertEquals(runtimeGraph.getState(), archivedGraph.getState());
	assertEquals(runtimeGraph.getFailureInfo().getExceptionAsString(), archivedGraph.getFailureInfo().getExceptionAsString());
	assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.CREATED), archivedGraph.getStatusTimestamp(JobStatus.CREATED));
	assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.RUNNING), archivedGraph.getStatusTimestamp(JobStatus.RUNNING));
	assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.FAILING), archivedGraph.getStatusTimestamp(JobStatus.FAILING));
	assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.FAILED), archivedGraph.getStatusTimestamp(JobStatus.FAILED));
	assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.CANCELLING), archivedGraph.getStatusTimestamp(JobStatus.CANCELLING));
	assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.CANCELED), archivedGraph.getStatusTimestamp(JobStatus.CANCELED));
	assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.FINISHED), archivedGraph.getStatusTimestamp(JobStatus.FINISHED));
	assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.RESTARTING), archivedGraph.getStatusTimestamp(JobStatus.RESTARTING));
	assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.SUSPENDED), archivedGraph.getStatusTimestamp(JobStatus.SUSPENDED));
	assertEquals(runtimeGraph.isStoppable(), archivedGraph.isStoppable());

	// -------------------------------------------------------------------------------------------------------------
	// CheckpointStats
	// -------------------------------------------------------------------------------------------------------------
	CheckpointStatsSnapshot runtimeSnapshot = runtimeGraph.getCheckpointStatsSnapshot();
	CheckpointStatsSnapshot archivedSnapshot = archivedGraph.getCheckpointStatsSnapshot();

	assertEquals(runtimeSnapshot.getSummaryStats().getEndToEndDurationStats().getAverage(), archivedSnapshot.getSummaryStats().getEndToEndDurationStats().getAverage());
	assertEquals(runtimeSnapshot.getSummaryStats().getEndToEndDurationStats().getMinimum(), archivedSnapshot.getSummaryStats().getEndToEndDurationStats().getMinimum());
	assertEquals(runtimeSnapshot.getSummaryStats().getEndToEndDurationStats().getMaximum(), archivedSnapshot.getSummaryStats().getEndToEndDurationStats().getMaximum());

	assertEquals(runtimeSnapshot.getSummaryStats().getStateSizeStats().getAverage(), archivedSnapshot.getSummaryStats().getStateSizeStats().getAverage());
	assertEquals(runtimeSnapshot.getSummaryStats().getStateSizeStats().getMinimum(), archivedSnapshot.getSummaryStats().getStateSizeStats().getMinimum());
	assertEquals(runtimeSnapshot.getSummaryStats().getStateSizeStats().getMaximum(), archivedSnapshot.getSummaryStats().getStateSizeStats().getMaximum());

	assertEquals(runtimeSnapshot.getCounts().getTotalNumberOfCheckpoints(), archivedSnapshot.getCounts().getTotalNumberOfCheckpoints());
	assertEquals(runtimeSnapshot.getCounts().getNumberOfCompletedCheckpoints(), archivedSnapshot.getCounts().getNumberOfCompletedCheckpoints());
	assertEquals(runtimeSnapshot.getCounts().getNumberOfInProgressCheckpoints(), archivedSnapshot.getCounts().getNumberOfInProgressCheckpoints());

	// -------------------------------------------------------------------------------------------------------------
	// ArchivedExecutionConfig
	// -------------------------------------------------------------------------------------------------------------
	ArchivedExecutionConfig runtimeConfig = runtimeGraph.getArchivedExecutionConfig();
	ArchivedExecutionConfig archivedConfig = archivedGraph.getArchivedExecutionConfig();

	assertEquals(runtimeConfig.getExecutionMode(), archivedConfig.getExecutionMode());
	assertEquals(runtimeConfig.getParallelism(), archivedConfig.getParallelism());
	assertEquals(runtimeConfig.getObjectReuseEnabled(), archivedConfig.getObjectReuseEnabled());
	assertEquals(runtimeConfig.getRestartStrategyDescription(), archivedConfig.getRestartStrategyDescription());
	assertNotNull(archivedConfig.getGlobalJobParameters().get("hello"));
	assertEquals(runtimeConfig.getGlobalJobParameters().get("hello"), archivedConfig.getGlobalJobParameters().get("hello"));

	// -------------------------------------------------------------------------------------------------------------
	// StringifiedAccumulators
	// -------------------------------------------------------------------------------------------------------------
	compareStringifiedAccumulators(runtimeGraph.getAccumulatorResultsStringified(), archivedGraph.getAccumulatorResultsStringified());
	compareSerializedAccumulators(runtimeGraph.getAccumulatorsSerialized(), archivedGraph.getAccumulatorsSerialized());

	// -------------------------------------------------------------------------------------------------------------
	// JobVertices
	// -------------------------------------------------------------------------------------------------------------
	Map<JobVertexID, ? extends AccessExecutionJobVertex> runtimeVertices = runtimeGraph.getAllVertices();
	Map<JobVertexID, ? extends AccessExecutionJobVertex> archivedVertices = archivedGraph.getAllVertices();

	for (Map.Entry<JobVertexID, ? extends AccessExecutionJobVertex> vertex : runtimeVertices.entrySet()) {
		compareExecutionJobVertex(vertex.getValue(), archivedVertices.get(vertex.getKey()));
	}

	Iterator<? extends AccessExecutionJobVertex> runtimeTopologicalVertices = runtimeGraph.getVerticesTopologically().iterator();
	Iterator<? extends AccessExecutionJobVertex> archiveTopologicaldVertices = archivedGraph.getVerticesTopologically().iterator();

	while (runtimeTopologicalVertices.hasNext()) {
		assertTrue(archiveTopologicaldVertices.hasNext());
		compareExecutionJobVertex(runtimeTopologicalVertices.next(), archiveTopologicaldVertices.next());
	}

	// -------------------------------------------------------------------------------------------------------------
	// ExecutionVertices
	// -------------------------------------------------------------------------------------------------------------
	Iterator<? extends AccessExecutionVertex> runtimeExecutionVertices = runtimeGraph.getAllExecutionVertices().iterator();
	Iterator<? extends AccessExecutionVertex> archivedExecutionVertices = archivedGraph.getAllExecutionVertices().iterator();

	while (runtimeExecutionVertices.hasNext()) {
		assertTrue(archivedExecutionVertices.hasNext());
		compareExecutionVertex(runtimeExecutionVertices.next(), archivedExecutionVertices.next());
	}
}
Example #27
Source File: AccessExecutionGraph.java    From flink with Apache License 2.0 2 votes vote down vote up 
/**
 * Returns the {@link ArchivedExecutionConfig} for this execution graph.
 *
 * @return execution config summary for this execution graph, or null in case of errors
 */
@Nullable
ArchivedExecutionConfig getArchivedExecutionConfig();
Example #28
Source File: AccessExecutionGraph.java    From flink with Apache License 2.0 2 votes vote down vote up 
/**
 * Returns the {@link ArchivedExecutionConfig} for this execution graph.
 *
 * @return execution config summary for this execution graph, or null in case of errors
 */
@Nullable
ArchivedExecutionConfig getArchivedExecutionConfig();
Example #29
Source File: AccessExecutionGraph.java    From Flink-CEPplus with Apache License 2.0 2 votes vote down vote up 
/**
 * Returns the {@link ArchivedExecutionConfig} for this execution graph.
 *
 * @return execution config summary for this execution graph, or null in case of errors
 */
@Nullable
ArchivedExecutionConfig getArchivedExecutionConfig();