org.apache.flink.runtime.executiongraph.restart.FixedDelayRestartStrategy Java Examples

private static ExecutionGraph createExecutionGraph(TaskManagerGateway gateway, int parallelism) throws Exception {
	final JobID jobId = new JobID();

	final JobVertex vertex = new JobVertex("vertex");
	vertex.setInvokableClass(NoOpInvokable.class);
	vertex.setParallelism(parallelism);

	final SlotProvider slotProvider = new SimpleSlotProvider(jobId, parallelism, gateway);

	ExecutionGraph simpleTestGraph = ExecutionGraphTestUtils.createSimpleTestGraph(
		jobId,
		slotProvider,
		new FixedDelayRestartStrategy(0, 0),
		vertex);
	simpleTestGraph.start(TestingComponentMainThreadExecutorServiceAdapter.forMainThread());
	return simpleTestGraph;
}
 

@Test
public void testNoRestartOnSuppressException() throws Exception {
	final ExecutionGraph eg = createExecutionGraph(new FixedDelayRestartStrategy(Integer.MAX_VALUE, 0)).f0;

	// Fail with unrecoverable Exception
	eg.getAllExecutionVertices().iterator().next().fail(
		new SuppressRestartsException(new Exception("Test Exception")));

	assertEquals(JobStatus.FAILING, eg.getState());

	completeCanceling(eg);

	eg.waitUntilTerminal();
	assertEquals(JobStatus.FAILED, eg.getState());

	RestartStrategy restartStrategy = eg.getRestartStrategy();
	assertTrue(restartStrategy instanceof FixedDelayRestartStrategy);

	assertEquals(0, ((FixedDelayRestartStrategy) restartStrategy).getCurrentRestartAttempt());
}
 

private static ExecutionGraph createExecutionGraph(TaskManagerGateway gateway, int parallelism) throws Exception {
	final JobID jobId = new JobID();

	final JobVertex vertex = new JobVertex("vertex");
	vertex.setInvokableClass(NoOpInvokable.class);
	vertex.setParallelism(parallelism);

	final SlotProvider slotProvider = new SimpleSlotProvider(jobId, parallelism, gateway);

	ExecutionGraph simpleTestGraph = ExecutionGraphTestUtils.createSimpleTestGraph(
		jobId,
		slotProvider,
		new FixedDelayRestartStrategy(0, 0),
		vertex);
	simpleTestGraph.start(ComponentMainThreadExecutorServiceAdapter.forMainThread());
	return simpleTestGraph;
}
 

@Test
public void testNoRestartOnSuppressException() throws Exception {
	try (SlotPool slotPool = createSlotPoolImpl()) {
		ExecutionGraph eg = TestingExecutionGraphBuilder.newBuilder()
			.setRestartStrategy(new FixedDelayRestartStrategy(Integer.MAX_VALUE, 0))
			.buildAndScheduleForExecution(slotPool);

		// Fail with unrecoverable Exception
		eg.getAllExecutionVertices().iterator().next().fail(
			new SuppressRestartsException(new Exception("Test Exception")));

		assertEquals(JobStatus.FAILING, eg.getState());

		completeCanceling(eg);

		eg.waitUntilTerminal();
		assertEquals(JobStatus.FAILED, eg.getState());

		RestartStrategy restartStrategy = eg.getRestartStrategy();
		assertTrue(restartStrategy instanceof FixedDelayRestartStrategy);

		assertEquals(0, ((FixedDelayRestartStrategy) restartStrategy).getCurrentRestartAttempt());
	}

}
 

/**
 * Tests that in a streaming use case where checkpointing is enabled, a
 * fixed delay with Integer.MAX_VALUE retries is instantiated if no other restart
 * strategy has been specified.
 */
@Test
public void testAutomaticRestartingWhenCheckpointing() throws Exception {
	// create savepoint data
	final long savepointId = 42L;
	final File savepointFile = createSavepoint(savepointId);

	// set savepoint settings
	final SavepointRestoreSettings savepointRestoreSettings = SavepointRestoreSettings.forPath(
		savepointFile.getAbsolutePath(),
		true);
	final JobGraph jobGraph = createJobGraphWithCheckpointing(savepointRestoreSettings);

	final StandaloneCompletedCheckpointStore completedCheckpointStore = new StandaloneCompletedCheckpointStore(1);
	final TestingCheckpointRecoveryFactory testingCheckpointRecoveryFactory = new TestingCheckpointRecoveryFactory(
		completedCheckpointStore,
		new StandaloneCheckpointIDCounter());
	haServices.setCheckpointRecoveryFactory(testingCheckpointRecoveryFactory);
	final JobMaster jobMaster = createJobMaster(
		new Configuration(),
		jobGraph,
		haServices,
		new TestingJobManagerSharedServicesBuilder()
			.setRestartStrategyFactory(RestartStrategyFactory.createRestartStrategyFactory(configuration))
			.build());

	RestartStrategy restartStrategy = jobMaster.getRestartStrategy();

	assertNotNull(restartStrategy);
	assertTrue(restartStrategy instanceof FixedDelayRestartStrategy);
}
 

/**
 * Creates a simple 2 vertex graph with a parallel source and a parallel target.
 */
private ExecutionGraph createTestGraph(int parallelism, boolean allToAll) throws Exception {

	JobVertex source = new JobVertex("source", sourceVertexId);
	source.setParallelism(parallelism);
	source.setInvokableClass(NoOpInvokable.class);

	JobVertex target = new JobVertex("source", targetVertexId);
	target.setParallelism(parallelism);
	target.setInvokableClass(NoOpInvokable.class);

	DistributionPattern connectionPattern = allToAll ? DistributionPattern.ALL_TO_ALL : DistributionPattern.POINTWISE;
	target.connectNewDataSetAsInput(source, connectionPattern, ResultPartitionType.PIPELINED);

	JobGraph testJob = new JobGraph(jobId, "test job", source, target);

	final Time timeout = Time.seconds(10L);
	return ExecutionGraphBuilder.buildGraph(
		null,
		testJob,
		new Configuration(),
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		mock(SlotProvider.class),
		getClass().getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		timeout,
		new FixedDelayRestartStrategy(10, 0L),
		new UnregisteredMetricsGroup(),
		1,
		VoidBlobWriter.getInstance(),
		timeout,
		log);
}
 

/**
 * Creates a simple 2 vertex graph with a parallel source and a parallel target.
 */
private ExecutionGraph createTestGraph(int parallelism, boolean allToAll) throws Exception {

	JobVertex source = new JobVertex("source", sourceVertexId);
	source.setParallelism(parallelism);
	source.setInvokableClass(NoOpInvokable.class);

	JobVertex target = new JobVertex("source", targetVertexId);
	target.setParallelism(parallelism);
	target.setInvokableClass(NoOpInvokable.class);

	DistributionPattern connectionPattern = allToAll ? DistributionPattern.ALL_TO_ALL : DistributionPattern.POINTWISE;
	target.connectNewDataSetAsInput(source, connectionPattern, ResultPartitionType.PIPELINED);

	JobGraph testJob = new JobGraph(jobId, "test job", source, target);

	final Time timeout = Time.seconds(10L);
	return ExecutionGraphBuilder.buildGraph(
		null,
		testJob,
		new Configuration(),
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		mock(SlotProvider.class),
		getClass().getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		timeout,
		new FixedDelayRestartStrategy(10, 0L),
		new UnregisteredMetricsGroup(),
		VoidBlobWriter.getInstance(),
		timeout,
		log,
		NettyShuffleMaster.INSTANCE,
		NoOpPartitionTracker.INSTANCE);
}
 

private ExecutionGraph createExecutionGraph(final JobGraph jobGraph) throws Exception {
	final ExecutionGraph executionGraph = new ExecutionGraphTestUtils.TestingExecutionGraphBuilder(jobGraph)
		.setRestartStrategy(new FixedDelayRestartStrategy(10, 0))
		.setFailoverStrategyFactory(AdaptedRestartPipelinedRegionStrategyNG::new)
		.setSlotProvider(new SimpleSlotProvider(jobGraph.getJobID(), 2))
		.build();

	enableCheckpointing(executionGraph);
	executionGraph.start(componentMainThreadExecutor);
	executionGraph.scheduleForExecution();
	manualMainThreadExecutor.triggerAll();
	return executionGraph;
}
 

private static ExecutionGraph createExecutionGraph(TaskManagerGateway gateway, int parallelism) throws Exception {
	final JobVertex vertex = new JobVertex("vertex");
	vertex.setInvokableClass(NoOpInvokable.class);
	vertex.setParallelism(parallelism);

	final SlotProvider slotProvider = new SimpleSlotProvider(parallelism, gateway);

	ExecutionGraph simpleTestGraph = ExecutionGraphTestUtils.createSimpleTestGraph(
		slotProvider,
		new FixedDelayRestartStrategy(0, 0),
		vertex);
	simpleTestGraph.start(ComponentMainThreadExecutorServiceAdapter.forMainThread());
	return simpleTestGraph;
}
 

/**
 * Creates a simple 2 vertex graph with a parallel source and a parallel target.
 */
private ExecutionGraph createTestGraph(int parallelism, boolean allToAll) throws Exception {

	JobVertex source = new JobVertex("source", sourceVertexId);
	source.setParallelism(parallelism);
	source.setInvokableClass(NoOpInvokable.class);

	JobVertex target = new JobVertex("source", targetVertexId);
	target.setParallelism(parallelism);
	target.setInvokableClass(NoOpInvokable.class);

	DistributionPattern connectionPattern = allToAll ? DistributionPattern.ALL_TO_ALL : DistributionPattern.POINTWISE;
	target.connectNewDataSetAsInput(source, connectionPattern, ResultPartitionType.PIPELINED);

	JobGraph testJob = new JobGraph(jobId, "test job", source, target);

	final Time timeout = Time.seconds(10L);
	return ExecutionGraphBuilder.buildGraph(
		null,
		testJob,
		new Configuration(),
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		mock(SlotProvider.class),
		getClass().getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		timeout,
		new FixedDelayRestartStrategy(10, 0L),
		new UnregisteredMetricsGroup(),
		VoidBlobWriter.getInstance(),
		timeout,
		log,
		NettyShuffleMaster.INSTANCE,
		NoOpJobMasterPartitionTracker.INSTANCE);
}
 

@Test
public void testNoRestartOnSuppressException() throws Exception {
	try (SlotPool slotPool = createSlotPoolImpl()) {
		ExecutionGraph eg = TestingExecutionGraphBuilder
			.newBuilder()
			.setJobGraph(createJobGraph())
			.setRestartStrategy(new FixedDelayRestartStrategy(Integer.MAX_VALUE, 0))
			.setSlotProvider(createSchedulerWithSlots(slotPool))
			.build();

		startAndScheduleExecutionGraph(eg);

		// Fail with unrecoverable Exception
		eg.getAllExecutionVertices().iterator().next().fail(
			new SuppressRestartsException(new Exception("Test Exception")));

		assertEquals(JobStatus.FAILING, eg.getState());

		completeCanceling(eg);

		eg.waitUntilTerminal();
		assertEquals(JobStatus.FAILED, eg.getState());

		RestartStrategy restartStrategy = eg.getRestartStrategy();
		assertTrue(restartStrategy instanceof FixedDelayRestartStrategy);

		assertEquals(0, ((FixedDelayRestartStrategy) restartStrategy).getCurrentRestartAttempt());
	}

}
 

private ExecutionGraph createExecutionGraph(final JobGraph jobGraph) throws Exception {
	return createExecutionGraph(jobGraph, new FixedDelayRestartStrategy(10, 0));
}