Java Code Examples for org.apache.flink.runtime.jobgraph.SavepointRestoreSettings#forPath()
The following examples show how to use
org.apache.flink.runtime.jobgraph.SavepointRestoreSettings#forPath() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: JarRunHandler.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
|
private SavepointRestoreSettings getSavepointRestoreSettings(
final @Nonnull HandlerRequest<JarRunRequestBody, JarRunMessageParameters> request)
throws RestHandlerException {
final JarRunRequestBody requestBody = request.getRequestBody();
final boolean allowNonRestoredState = fromRequestBodyOrQueryParameter(
requestBody.getAllowNonRestoredState(),
() -> getQueryParameter(request, AllowNonRestoredStateQueryParameter.class),
false,
log);
final String savepointPath = fromRequestBodyOrQueryParameter(
emptyToNull(requestBody.getSavepointPath()),
() -> emptyToNull(getQueryParameter(request, SavepointPathQueryParameter.class)),
null,
log);
final SavepointRestoreSettings savepointRestoreSettings;
if (savepointPath != null) {
savepointRestoreSettings = SavepointRestoreSettings.forPath(
savepointPath,
allowNonRestoredState);
} else {
savepointRestoreSettings = SavepointRestoreSettings.none();
}
return savepointRestoreSettings;
}
Example 2
| Source File: JarRunHandler.java From flink with Apache License 2.0 | 6 votes |
|
private SavepointRestoreSettings getSavepointRestoreSettings(
final @Nonnull HandlerRequest<JarRunRequestBody, JarRunMessageParameters> request)
throws RestHandlerException {
final JarRunRequestBody requestBody = request.getRequestBody();
final boolean allowNonRestoredState = fromRequestBodyOrQueryParameter(
requestBody.getAllowNonRestoredState(),
() -> getQueryParameter(request, AllowNonRestoredStateQueryParameter.class),
false,
log);
final String savepointPath = fromRequestBodyOrQueryParameter(
emptyToNull(requestBody.getSavepointPath()),
() -> emptyToNull(getQueryParameter(request, SavepointPathQueryParameter.class)),
null,
log);
final SavepointRestoreSettings savepointRestoreSettings;
if (savepointPath != null) {
savepointRestoreSettings = SavepointRestoreSettings.forPath(
savepointPath,
allowNonRestoredState);
} else {
savepointRestoreSettings = SavepointRestoreSettings.none();
}
return savepointRestoreSettings;
}
Example 3
| Source File: ClassPathPackagedProgramRetrieverTest.java From flink with Apache License 2.0 | 6 votes |
|
@Test
public void testSavepointRestoreSettings() throws FlinkException, IOException, ProgramInvocationException {
final Configuration configuration = new Configuration();
final SavepointRestoreSettings savepointRestoreSettings = SavepointRestoreSettings.forPath("foobar", true);
final JobID jobId = new JobID();
configuration.setString(PipelineOptionsInternal.PIPELINE_FIXED_JOB_ID, jobId.toHexString());
SavepointRestoreSettings.toConfiguration(savepointRestoreSettings, configuration);
final ClassPathPackagedProgramRetriever retrieverUnderTest =
ClassPathPackagedProgramRetriever.newBuilder(PROGRAM_ARGUMENTS)
.setJobClassName(TestJob.class.getCanonicalName())
.build();
final JobGraph jobGraph = retrieveJobGraph(retrieverUnderTest, configuration);
assertThat(jobGraph.getSavepointRestoreSettings(), is(equalTo(savepointRestoreSettings)));
assertEquals(jobGraph.getJobID(), jobId);
}
Example 4
| Source File: JarRunHandler.java From flink with Apache License 2.0 | 6 votes |
|
private SavepointRestoreSettings getSavepointRestoreSettings(
final @Nonnull HandlerRequest<JarRunRequestBody, JarRunMessageParameters> request)
throws RestHandlerException {
final JarRunRequestBody requestBody = request.getRequestBody();
final boolean allowNonRestoredState = fromRequestBodyOrQueryParameter(
requestBody.getAllowNonRestoredState(),
() -> getQueryParameter(request, AllowNonRestoredStateQueryParameter.class),
false,
log);
final String savepointPath = fromRequestBodyOrQueryParameter(
emptyToNull(requestBody.getSavepointPath()),
() -> emptyToNull(getQueryParameter(request, SavepointPathQueryParameter.class)),
null,
log);
final SavepointRestoreSettings savepointRestoreSettings;
if (savepointPath != null) {
savepointRestoreSettings = SavepointRestoreSettings.forPath(
savepointPath,
allowNonRestoredState);
} else {
savepointRestoreSettings = SavepointRestoreSettings.none();
}
return savepointRestoreSettings;
}
Example 5
| Source File: ClassPathJobGraphRetrieverTest.java From flink with Apache License 2.0 | 6 votes |
|
@Test
public void testSavepointRestoreSettings() throws FlinkException {
final Configuration configuration = new Configuration();
final SavepointRestoreSettings savepointRestoreSettings = SavepointRestoreSettings.forPath("foobar", true);
final JobID jobId = new JobID();
final ClassPathJobGraphRetriever classPathJobGraphRetriever = new ClassPathJobGraphRetriever(
jobId,
savepointRestoreSettings,
PROGRAM_ARGUMENTS,
TestJob.class.getCanonicalName());
final JobGraph jobGraph = classPathJobGraphRetriever.retrieveJobGraph(configuration);
assertThat(jobGraph.getSavepointRestoreSettings(), is(equalTo(savepointRestoreSettings)));
assertEquals(jobGraph.getJobID(), jobId);
}
Example 6
| Source File: ClassPathJobGraphRetrieverTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
|
@Test
public void testSavepointRestoreSettings() throws FlinkException {
final Configuration configuration = new Configuration();
final SavepointRestoreSettings savepointRestoreSettings = SavepointRestoreSettings.forPath("foobar", true);
final JobID jobId = new JobID();
final ClassPathJobGraphRetriever classPathJobGraphRetriever = new ClassPathJobGraphRetriever(
jobId,
savepointRestoreSettings,
PROGRAM_ARGUMENTS,
TestJob.class.getCanonicalName());
final JobGraph jobGraph = classPathJobGraphRetriever.retrieveJobGraph(configuration);
assertThat(jobGraph.getSavepointRestoreSettings(), is(equalTo(savepointRestoreSettings)));
assertEquals(jobGraph.getJobID(), jobId);
}
Example 7
| Source File: CliFrontendParser.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
public static SavepointRestoreSettings createSavepointRestoreSettings(CommandLine commandLine) {
if (commandLine.hasOption(SAVEPOINT_PATH_OPTION.getOpt())) {
String savepointPath = commandLine.getOptionValue(SAVEPOINT_PATH_OPTION.getOpt());
boolean allowNonRestoredState = commandLine.hasOption(SAVEPOINT_ALLOW_NON_RESTORED_OPTION.getOpt());
return SavepointRestoreSettings.forPath(savepointPath, allowNonRestoredState);
} else {
return SavepointRestoreSettings.none();
}
}
Example 8
| Source File: StreamingJobGraphGeneratorTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void generatorForwardsSavepointRestoreSettings() {
StreamGraph streamGraph = new StreamGraph(
new ExecutionConfig(),
new CheckpointConfig(),
SavepointRestoreSettings.forPath("hello"));
JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(streamGraph);
SavepointRestoreSettings savepointRestoreSettings = jobGraph.getSavepointRestoreSettings();
assertThat(savepointRestoreSettings.getRestorePath(), is("hello"));
}
Example 9
| Source File: JobMasterTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
/**
* Tests that in a streaming use case where checkpointing is enabled, a
* fixed delay with Integer.MAX_VALUE retries is instantiated if no other restart
* strategy has been specified.
*/
@Test
public void testAutomaticRestartingWhenCheckpointing() throws Exception {
// create savepoint data
final long savepointId = 42L;
final File savepointFile = createSavepoint(savepointId);
// set savepoint settings
final SavepointRestoreSettings savepointRestoreSettings = SavepointRestoreSettings.forPath(
savepointFile.getAbsolutePath(),
true);
final JobGraph jobGraph = createJobGraphWithCheckpointing(savepointRestoreSettings);
final StandaloneCompletedCheckpointStore completedCheckpointStore = new StandaloneCompletedCheckpointStore(1);
final TestingCheckpointRecoveryFactory testingCheckpointRecoveryFactory = new TestingCheckpointRecoveryFactory(
completedCheckpointStore,
new StandaloneCheckpointIDCounter());
haServices.setCheckpointRecoveryFactory(testingCheckpointRecoveryFactory);
final JobMaster jobMaster = createJobMaster(
new Configuration(),
jobGraph,
haServices,
new TestingJobManagerSharedServicesBuilder()
.setRestartStrategyFactory(RestartStrategyFactory.createRestartStrategyFactory(configuration))
.build());
RestartStrategy restartStrategy = jobMaster.getRestartStrategy();
assertNotNull(restartStrategy);
assertTrue(restartStrategy instanceof FixedDelayRestartStrategy);
}
Example 10
| Source File: JobMasterTest.java From flink with Apache License 2.0 | 5 votes |
|
/**
* Tests that a JobMaster will restore the given JobGraph from its savepoint upon
* initial submission.
*/
@Test
public void testRestoringFromSavepoint() throws Exception {
// create savepoint data
final long savepointId = 42L;
final File savepointFile = createSavepoint(savepointId);
// set savepoint settings
final SavepointRestoreSettings savepointRestoreSettings = SavepointRestoreSettings.forPath(
savepointFile.getAbsolutePath(),
true);
final JobGraph jobGraph = createJobGraphWithCheckpointing(savepointRestoreSettings);
final StandaloneCompletedCheckpointStore completedCheckpointStore = new StandaloneCompletedCheckpointStore(1);
final TestingCheckpointRecoveryFactory testingCheckpointRecoveryFactory = new TestingCheckpointRecoveryFactory(completedCheckpointStore, new StandaloneCheckpointIDCounter());
haServices.setCheckpointRecoveryFactory(testingCheckpointRecoveryFactory);
final JobMaster jobMaster = createJobMaster(
configuration,
jobGraph,
haServices,
new TestingJobManagerSharedServicesBuilder().build());
try {
// starting the JobMaster should have read the savepoint
final CompletedCheckpoint savepointCheckpoint = completedCheckpointStore.getLatestCheckpoint(false);
assertThat(savepointCheckpoint, Matchers.notNullValue());
assertThat(savepointCheckpoint.getCheckpointID(), is(savepointId));
} finally {
RpcUtils.terminateRpcEndpoint(jobMaster, testingTimeout);
}
}
Example 11
| Source File: JobMasterTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
/**
* Tests that a JobMaster will restore the given JobGraph from its savepoint upon
* initial submission.
*/
@Test
public void testRestoringFromSavepoint() throws Exception {
// create savepoint data
final long savepointId = 42L;
final File savepointFile = createSavepoint(savepointId);
// set savepoint settings
final SavepointRestoreSettings savepointRestoreSettings = SavepointRestoreSettings.forPath(
savepointFile.getAbsolutePath(),
true);
final JobGraph jobGraph = createJobGraphWithCheckpointing(savepointRestoreSettings);
final StandaloneCompletedCheckpointStore completedCheckpointStore = new StandaloneCompletedCheckpointStore(1);
final TestingCheckpointRecoveryFactory testingCheckpointRecoveryFactory = new TestingCheckpointRecoveryFactory(completedCheckpointStore, new StandaloneCheckpointIDCounter());
haServices.setCheckpointRecoveryFactory(testingCheckpointRecoveryFactory);
final JobMaster jobMaster = createJobMaster(
configuration,
jobGraph,
haServices,
new TestingJobManagerSharedServicesBuilder().build());
try {
// starting the JobMaster should have read the savepoint
final CompletedCheckpoint savepointCheckpoint = completedCheckpointStore.getLatestCheckpoint();
assertThat(savepointCheckpoint, Matchers.notNullValue());
assertThat(savepointCheckpoint.getCheckpointID(), is(savepointId));
} finally {
RpcUtils.terminateRpcEndpoint(jobMaster, testingTimeout);
}
}
Example 12
| Source File: FlinkRequiresStableInputTest.java From beam with Apache License 2.0 | 5 votes |
|
private JobID restoreFromSavepoint(Pipeline pipeline, String savepointDir)
throws ExecutionException, InterruptedException {
JobGraph jobGraph = getJobGraph(pipeline);
SavepointRestoreSettings savepointSettings = SavepointRestoreSettings.forPath(savepointDir);
jobGraph.setSavepointRestoreSettings(savepointSettings);
return flinkCluster.submitJob(jobGraph).get().getJobID();
}
Example 13
| Source File: FlinkSavepointTest.java From beam with Apache License 2.0 | 5 votes |
|
private void restoreFromSavepointLegacy(Pipeline pipeline, String savepointDir)
throws ExecutionException, InterruptedException {
JobGraph jobGraph = getJobGraph(pipeline);
SavepointRestoreSettings savepointSettings = SavepointRestoreSettings.forPath(savepointDir);
jobGraph.setSavepointRestoreSettings(savepointSettings);
flinkCluster.submitJob(jobGraph).get();
}
Example 14
| Source File: JobMasterTest.java From flink with Apache License 2.0 | 5 votes |
|
/**
* Tests that a JobMaster will restore the given JobGraph from its savepoint upon
* initial submission.
*/
@Test
public void testRestoringFromSavepoint() throws Exception {
// create savepoint data
final long savepointId = 42L;
final File savepointFile = createSavepoint(savepointId);
// set savepoint settings
final SavepointRestoreSettings savepointRestoreSettings = SavepointRestoreSettings.forPath(
savepointFile.getAbsolutePath(),
true);
final JobGraph jobGraph = createJobGraphWithCheckpointing(savepointRestoreSettings);
final StandaloneCompletedCheckpointStore completedCheckpointStore = new StandaloneCompletedCheckpointStore(1);
final TestingCheckpointRecoveryFactory testingCheckpointRecoveryFactory = new TestingCheckpointRecoveryFactory(completedCheckpointStore, new StandaloneCheckpointIDCounter());
haServices.setCheckpointRecoveryFactory(testingCheckpointRecoveryFactory);
final JobMaster jobMaster = createJobMaster(
configuration,
jobGraph,
haServices,
new TestingJobManagerSharedServicesBuilder().build());
try {
// starting the JobMaster should have read the savepoint
final CompletedCheckpoint savepointCheckpoint = completedCheckpointStore.getLatestCheckpoint(false);
assertThat(savepointCheckpoint, Matchers.notNullValue());
assertThat(savepointCheckpoint.getCheckpointID(), is(savepointId));
} finally {
RpcUtils.terminateRpcEndpoint(jobMaster, testingTimeout);
}
}
Example 15
| Source File: StandaloneApplicationClusterConfigurationParserFactoryTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testEntrypointClusterConfigurationToConfigurationParsing() throws FlinkParseException {
final JobID jobID = JobID.generate();
final SavepointRestoreSettings savepointRestoreSettings = SavepointRestoreSettings.forPath("/test/savepoint/path", true);
final String key = DeploymentOptions.TARGET.key();
final String value = "testDynamicExecutorConfig";
final int restPort = 1234;
final String arg1 = "arg1";
final String arg2 = "arg2";
final String[] args = {
"--configDir", confDirPath,
"--job-id", jobID.toHexString(),
"--fromSavepoint", savepointRestoreSettings.getRestorePath(),
"--allowNonRestoredState",
"--webui-port", String.valueOf(restPort),
"--job-classname", JOB_CLASS_NAME,
String.format("-D%s=%s", key, value),
arg1, arg2};
final StandaloneApplicationClusterConfiguration clusterConfiguration = commandLineParser.parse(args);
assertThat(clusterConfiguration.getJobClassName(), is(equalTo(JOB_CLASS_NAME)));
assertThat(clusterConfiguration.getArgs(), arrayContaining(arg1, arg2));
final Configuration configuration = StandaloneApplicationClusterEntryPoint
.loadConfigurationFromClusterConfig(clusterConfiguration);
final String strJobId = configuration.get(PipelineOptionsInternal.PIPELINE_FIXED_JOB_ID);
assertThat(JobID.fromHexString(strJobId), is(equalTo(jobID)));
assertThat(SavepointRestoreSettings.fromConfiguration(configuration), is(equalTo(savepointRestoreSettings)));
assertThat(configuration.get(RestOptions.PORT), is(equalTo(restPort)));
assertThat(configuration.get(DeploymentOptions.TARGET), is(equalTo(value)));
}
Example 16
| Source File: JobMasterTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
/**
* Tests that an existing checkpoint will have precedence over an savepoint.
*/
@Test
public void testCheckpointPrecedesSavepointRecovery() throws Exception {
// create savepoint data
final long savepointId = 42L;
final File savepointFile = createSavepoint(savepointId);
// set savepoint settings
final SavepointRestoreSettings savepointRestoreSettings = SavepointRestoreSettings.forPath("" +
savepointFile.getAbsolutePath(),
true);
final JobGraph jobGraph = createJobGraphWithCheckpointing(savepointRestoreSettings);
final long checkpointId = 1L;
final CompletedCheckpoint completedCheckpoint = new CompletedCheckpoint(
jobGraph.getJobID(),
checkpointId,
1L,
1L,
Collections.emptyMap(),
null,
CheckpointProperties.forCheckpoint(CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION),
new DummyCheckpointStorageLocation());
final StandaloneCompletedCheckpointStore completedCheckpointStore = new StandaloneCompletedCheckpointStore(1);
completedCheckpointStore.addCheckpoint(completedCheckpoint);
final TestingCheckpointRecoveryFactory testingCheckpointRecoveryFactory = new TestingCheckpointRecoveryFactory(completedCheckpointStore, new StandaloneCheckpointIDCounter());
haServices.setCheckpointRecoveryFactory(testingCheckpointRecoveryFactory);
final JobMaster jobMaster = createJobMaster(
configuration,
jobGraph,
haServices,
new TestingJobManagerSharedServicesBuilder().build());
try {
// starting the JobMaster should have read the savepoint
final CompletedCheckpoint savepointCheckpoint = completedCheckpointStore.getLatestCheckpoint();
assertThat(savepointCheckpoint, Matchers.notNullValue());
assertThat(savepointCheckpoint.getCheckpointID(), is(checkpointId));
} finally {
RpcUtils.terminateRpcEndpoint(jobMaster, testingTimeout);
}
}
Example 17
| Source File: JobMasterTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
/**
* Tests that a JobMaster will only restore a modified JobGraph if non
* restored state is allowed.
*/
@Test
public void testRestoringModifiedJobFromSavepoint() throws Exception {
// create savepoint data
final long savepointId = 42L;
final OperatorID operatorID = new OperatorID();
final File savepointFile = createSavepointWithOperatorState(savepointId, operatorID);
// set savepoint settings which don't allow non restored state
final SavepointRestoreSettings savepointRestoreSettings = SavepointRestoreSettings.forPath(
savepointFile.getAbsolutePath(),
false);
// create a new operator
final JobVertex jobVertex = new JobVertex("New operator");
jobVertex.setInvokableClass(NoOpInvokable.class);
final JobGraph jobGraphWithNewOperator = createJobGraphFromJobVerticesWithCheckpointing(savepointRestoreSettings, jobVertex);
final StandaloneCompletedCheckpointStore completedCheckpointStore = new StandaloneCompletedCheckpointStore(1);
final TestingCheckpointRecoveryFactory testingCheckpointRecoveryFactory = new TestingCheckpointRecoveryFactory(completedCheckpointStore, new StandaloneCheckpointIDCounter());
haServices.setCheckpointRecoveryFactory(testingCheckpointRecoveryFactory);
try {
createJobMaster(
configuration,
jobGraphWithNewOperator,
haServices,
new TestingJobManagerSharedServicesBuilder().build());
fail("Should fail because we cannot resume the changed JobGraph from the savepoint.");
} catch (IllegalStateException expected) {
// that was expected :-)
}
// allow for non restored state
jobGraphWithNewOperator.setSavepointRestoreSettings(
SavepointRestoreSettings.forPath(
savepointFile.getAbsolutePath(),
true));
final JobMaster jobMaster = createJobMaster(
configuration,
jobGraphWithNewOperator,
haServices,
new TestingJobManagerSharedServicesBuilder().build());
try {
// starting the JobMaster should have read the savepoint
final CompletedCheckpoint savepointCheckpoint = completedCheckpointStore.getLatestCheckpoint();
assertThat(savepointCheckpoint, Matchers.notNullValue());
assertThat(savepointCheckpoint.getCheckpointID(), is(savepointId));
} finally {
RpcUtils.terminateRpcEndpoint(jobMaster, testingTimeout);
}
}
Example 18
| Source File: JobMasterTest.java From flink with Apache License 2.0 | 4 votes |
|
/**
* Tests that a JobMaster will only restore a modified JobGraph if non
* restored state is allowed.
*/
@Test
public void testRestoringModifiedJobFromSavepoint() throws Exception {
// create savepoint data
final long savepointId = 42L;
final OperatorID operatorID = new OperatorID();
final File savepointFile = createSavepointWithOperatorState(savepointId, operatorID);
// set savepoint settings which don't allow non restored state
final SavepointRestoreSettings savepointRestoreSettings = SavepointRestoreSettings.forPath(
savepointFile.getAbsolutePath(),
false);
// create a new operator
final JobVertex jobVertex = new JobVertex("New operator");
jobVertex.setInvokableClass(NoOpInvokable.class);
final JobGraph jobGraphWithNewOperator = createJobGraphFromJobVerticesWithCheckpointing(savepointRestoreSettings, jobVertex);
final StandaloneCompletedCheckpointStore completedCheckpointStore = new StandaloneCompletedCheckpointStore(1);
final TestingCheckpointRecoveryFactory testingCheckpointRecoveryFactory = new TestingCheckpointRecoveryFactory(completedCheckpointStore, new StandaloneCheckpointIDCounter());
haServices.setCheckpointRecoveryFactory(testingCheckpointRecoveryFactory);
try {
createJobMaster(
configuration,
jobGraphWithNewOperator,
haServices,
new TestingJobManagerSharedServicesBuilder().build());
fail("Should fail because we cannot resume the changed JobGraph from the savepoint.");
} catch (IllegalStateException expected) {
// that was expected :-)
}
// allow for non restored state
jobGraphWithNewOperator.setSavepointRestoreSettings(
SavepointRestoreSettings.forPath(
savepointFile.getAbsolutePath(),
true));
final JobMaster jobMaster = createJobMaster(
configuration,
jobGraphWithNewOperator,
haServices,
new TestingJobManagerSharedServicesBuilder().build());
try {
// starting the JobMaster should have read the savepoint
final CompletedCheckpoint savepointCheckpoint = completedCheckpointStore.getLatestCheckpoint(false);
assertThat(savepointCheckpoint, Matchers.notNullValue());
assertThat(savepointCheckpoint.getCheckpointID(), is(savepointId));
} finally {
RpcUtils.terminateRpcEndpoint(jobMaster, testingTimeout);
}
}
Example 19
| Source File: JobMasterTest.java From flink with Apache License 2.0 | 4 votes |
|
/**
* Tests that an existing checkpoint will have precedence over an savepoint.
*/
@Test
public void testCheckpointPrecedesSavepointRecovery() throws Exception {
// create savepoint data
final long savepointId = 42L;
final File savepointFile = createSavepoint(savepointId);
// set savepoint settings
final SavepointRestoreSettings savepointRestoreSettings = SavepointRestoreSettings.forPath("" +
savepointFile.getAbsolutePath(),
true);
final JobGraph jobGraph = createJobGraphWithCheckpointing(savepointRestoreSettings);
final long checkpointId = 1L;
final CompletedCheckpoint completedCheckpoint = new CompletedCheckpoint(
jobGraph.getJobID(),
checkpointId,
1L,
1L,
Collections.emptyMap(),
null,
CheckpointProperties.forCheckpoint(CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION),
new DummyCheckpointStorageLocation());
final StandaloneCompletedCheckpointStore completedCheckpointStore = new StandaloneCompletedCheckpointStore(1);
completedCheckpointStore.addCheckpoint(completedCheckpoint);
final TestingCheckpointRecoveryFactory testingCheckpointRecoveryFactory = new TestingCheckpointRecoveryFactory(completedCheckpointStore, new StandaloneCheckpointIDCounter());
haServices.setCheckpointRecoveryFactory(testingCheckpointRecoveryFactory);
final JobMaster jobMaster = createJobMaster(
configuration,
jobGraph,
haServices,
new TestingJobManagerSharedServicesBuilder().build());
try {
// starting the JobMaster should have read the savepoint
final CompletedCheckpoint savepointCheckpoint = completedCheckpointStore.getLatestCheckpoint(false);
assertThat(savepointCheckpoint, Matchers.notNullValue());
assertThat(savepointCheckpoint.getCheckpointID(), is(checkpointId));
} finally {
RpcUtils.terminateRpcEndpoint(jobMaster, testingTimeout);
}
}
Example 20
| Source File: JobMasterTest.java From flink with Apache License 2.0 | 4 votes |
|
/**
* Tests that a JobMaster will only restore a modified JobGraph if non
* restored state is allowed.
*/
@Test
public void testRestoringModifiedJobFromSavepoint() throws Exception {
// create savepoint data
final long savepointId = 42L;
final OperatorID operatorID = new OperatorID();
final File savepointFile = createSavepointWithOperatorState(savepointId, operatorID);
// set savepoint settings which don't allow non restored state
final SavepointRestoreSettings savepointRestoreSettings = SavepointRestoreSettings.forPath(
savepointFile.getAbsolutePath(),
false);
// create a new operator
final JobVertex jobVertex = new JobVertex("New operator");
jobVertex.setInvokableClass(NoOpInvokable.class);
final JobGraph jobGraphWithNewOperator = createJobGraphFromJobVerticesWithCheckpointing(savepointRestoreSettings, jobVertex);
final StandaloneCompletedCheckpointStore completedCheckpointStore = new StandaloneCompletedCheckpointStore(1);
final TestingCheckpointRecoveryFactory testingCheckpointRecoveryFactory = new TestingCheckpointRecoveryFactory(completedCheckpointStore, new StandaloneCheckpointIDCounter());
haServices.setCheckpointRecoveryFactory(testingCheckpointRecoveryFactory);
try {
createJobMaster(
configuration,
jobGraphWithNewOperator,
haServices,
new TestingJobManagerSharedServicesBuilder().build());
fail("Should fail because we cannot resume the changed JobGraph from the savepoint.");
} catch (IllegalStateException expected) {
// that was expected :-)
}
// allow for non restored state
jobGraphWithNewOperator.setSavepointRestoreSettings(
SavepointRestoreSettings.forPath(
savepointFile.getAbsolutePath(),
true));
final JobMaster jobMaster = createJobMaster(
configuration,
jobGraphWithNewOperator,
haServices,
new TestingJobManagerSharedServicesBuilder().build());
try {
// starting the JobMaster should have read the savepoint
final CompletedCheckpoint savepointCheckpoint = completedCheckpointStore.getLatestCheckpoint(false);
assertThat(savepointCheckpoint, Matchers.notNullValue());
assertThat(savepointCheckpoint.getCheckpointID(), is(savepointId));
} finally {
RpcUtils.terminateRpcEndpoint(jobMaster, testingTimeout);
}
}
