org.apache.flink.runtime.taskmanager.TestCheckpointResponder Java Examples
The following examples show how to use
org.apache.flink.runtime.taskmanager.TestCheckpointResponder.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TaskSubmissionTestEnvironment.java From flink with Apache License 2.0 | 6 votes |
static void registerJobMasterConnection( JobTable jobTable, JobID jobId, RpcService testingRpcService, JobMasterGateway jobMasterGateway, TaskManagerActions taskManagerActions, Time timeout, MainThreadExecutable mainThreadExecutable) { mainThreadExecutable.runAsync(() -> { final JobTable.Job job = jobTable.getOrCreateJob(jobId, () -> TestingJobServices.newBuilder().build()); job.connect( ResourceID.generate(), jobMasterGateway, taskManagerActions, new TestCheckpointResponder(), new TestGlobalAggregateManager(), new RpcResultPartitionConsumableNotifier(jobMasterGateway, testingRpcService.getExecutor(), timeout), TestingPartitionProducerStateChecker.newBuilder() .setPartitionProducerStateFunction((jobID, intermediateDataSetID, resultPartitionID) -> CompletableFuture.completedFuture(ExecutionState.RUNNING)) .build()); }); }
Example #2
Source File: TestTaskStateManager.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
public TestTaskStateManager(LocalRecoveryConfig localRecoveryConfig) { this( new JobID(), new ExecutionAttemptID(), new TestCheckpointResponder(), localRecoveryConfig); }
Example #3
Source File: TestTaskStateManager.java From flink with Apache License 2.0 | 5 votes |
public TestTaskStateManager(LocalRecoveryConfig localRecoveryConfig) { this( new JobID(), new ExecutionAttemptID(), new TestCheckpointResponder(), localRecoveryConfig); }
Example #4
Source File: TestTaskStateManager.java From flink with Apache License 2.0 | 5 votes |
public TestTaskStateManager(LocalRecoveryConfig localRecoveryConfig) { this( new JobID(), new ExecutionAttemptID(), new TestCheckpointResponder(), localRecoveryConfig); }
Example #5
Source File: StreamTaskStateInitializerImplTest.java From flink with Apache License 2.0 | 5 votes |
private StreamTaskStateInitializer streamTaskStateManager( StateBackend stateBackend, JobManagerTaskRestore jobManagerTaskRestore, boolean createTimerServiceManager) { JobID jobID = new JobID(42L, 43L); ExecutionAttemptID executionAttemptID = new ExecutionAttemptID(23L, 24L); TestCheckpointResponder checkpointResponderMock = new TestCheckpointResponder(); TaskLocalStateStore taskLocalStateStore = new TestTaskLocalStateStore(); TaskStateManager taskStateManager = TaskStateManagerImplTest.taskStateManager( jobID, executionAttemptID, checkpointResponderMock, jobManagerTaskRestore, taskLocalStateStore); DummyEnvironment dummyEnvironment = new DummyEnvironment("test-task", 1, 0); dummyEnvironment.setTaskStateManager(taskStateManager); if (createTimerServiceManager) { return new StreamTaskStateInitializerImpl( dummyEnvironment, stateBackend); } else { return new StreamTaskStateInitializerImpl( dummyEnvironment, stateBackend) { @Override protected <K> InternalTimeServiceManager<K> internalTimeServiceManager( AbstractKeyedStateBackend<K> keyedStatedBackend, KeyContext keyContext, ProcessingTimeService processingTimeService, Iterable<KeyGroupStatePartitionStreamProvider> rawKeyedStates) throws Exception { return null; } }; } }
Example #6
Source File: TaskStateManagerImplTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
/** * This tests if the {@link TaskStateManager} properly returns the the subtask local state dir from the * corresponding {@link TaskLocalStateStoreImpl}. */ @Test public void testForwardingSubtaskLocalStateBaseDirFromLocalStateStore() throws IOException { JobID jobID = new JobID(42L, 43L); AllocationID allocationID = new AllocationID(4711L, 23L); JobVertexID jobVertexID = new JobVertexID(12L, 34L); ExecutionAttemptID executionAttemptID = new ExecutionAttemptID(23L, 24L); TestCheckpointResponder checkpointResponderMock = new TestCheckpointResponder(); Executor directExecutor = Executors.directExecutor(); TemporaryFolder tmpFolder = new TemporaryFolder(); try { tmpFolder.create(); File[] allocBaseDirs = new File[]{tmpFolder.newFolder(), tmpFolder.newFolder(), tmpFolder.newFolder()}; LocalRecoveryDirectoryProviderImpl directoryProvider = new LocalRecoveryDirectoryProviderImpl(allocBaseDirs, jobID, jobVertexID, 0); LocalRecoveryConfig localRecoveryConfig = new LocalRecoveryConfig(true, directoryProvider); TaskLocalStateStore taskLocalStateStore = new TaskLocalStateStoreImpl(jobID, allocationID, jobVertexID, 13, localRecoveryConfig, directExecutor); TaskStateManager taskStateManager = taskStateManager( jobID, executionAttemptID, checkpointResponderMock, null, taskLocalStateStore); LocalRecoveryConfig localRecoveryConfFromTaskLocalStateStore = taskLocalStateStore.getLocalRecoveryConfig(); LocalRecoveryConfig localRecoveryConfFromTaskStateManager = taskStateManager.createLocalRecoveryConfig(); for (int i = 0; i < 10; ++i) { Assert.assertEquals(allocBaseDirs[i % allocBaseDirs.length], localRecoveryConfFromTaskLocalStateStore.getLocalStateDirectoryProvider().allocationBaseDirectory(i)); Assert.assertEquals(allocBaseDirs[i % allocBaseDirs.length], localRecoveryConfFromTaskStateManager.getLocalStateDirectoryProvider().allocationBaseDirectory(i)); } Assert.assertEquals( localRecoveryConfFromTaskLocalStateStore.isLocalRecoveryEnabled(), localRecoveryConfFromTaskStateManager.isLocalRecoveryEnabled()); } finally { tmpFolder.delete(); } }
Example #7
Source File: StreamTaskStateInitializerImplTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
private StreamTaskStateInitializer streamTaskStateManager( StateBackend stateBackend, JobManagerTaskRestore jobManagerTaskRestore, boolean createTimerServiceManager) { JobID jobID = new JobID(42L, 43L); ExecutionAttemptID executionAttemptID = new ExecutionAttemptID(23L, 24L); TestCheckpointResponder checkpointResponderMock = new TestCheckpointResponder(); TaskLocalStateStore taskLocalStateStore = new TestTaskLocalStateStore(); TaskStateManager taskStateManager = TaskStateManagerImplTest.taskStateManager( jobID, executionAttemptID, checkpointResponderMock, jobManagerTaskRestore, taskLocalStateStore); DummyEnvironment dummyEnvironment = new DummyEnvironment("test-task", 1, 0); dummyEnvironment.setTaskStateManager(taskStateManager); ProcessingTimeService processingTimeService = new TestProcessingTimeService(); if (createTimerServiceManager) { return new StreamTaskStateInitializerImpl( dummyEnvironment, stateBackend, processingTimeService); } else { return new StreamTaskStateInitializerImpl( dummyEnvironment, stateBackend, processingTimeService) { @Override protected <K> InternalTimeServiceManager<K> internalTimeServiceManager( AbstractKeyedStateBackend<K> keyedStatedBackend, KeyContext keyContext, Iterable<KeyGroupStatePartitionStreamProvider> rawKeyedStates) throws Exception { return null; } }; } }
Example #8
Source File: LocalStateForwardingTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
/** * This tests that state that was reported to the {@link org.apache.flink.runtime.state.TaskStateManager} is also * reported to {@link org.apache.flink.runtime.taskmanager.CheckpointResponder} and {@link TaskLocalStateStoreImpl}. */ @Test public void testReportingFromTaskStateManagerToResponderAndTaskLocalStateStore() throws Exception { final JobID jobID = new JobID(); final AllocationID allocationID = new AllocationID(); final ExecutionAttemptID executionAttemptID = new ExecutionAttemptID(); final CheckpointMetaData checkpointMetaData = new CheckpointMetaData(42L, 4711L); final CheckpointMetrics checkpointMetrics = new CheckpointMetrics(); final int subtaskIdx = 42; JobVertexID jobVertexID = new JobVertexID(); TaskStateSnapshot jmSnapshot = new TaskStateSnapshot(); TaskStateSnapshot tmSnapshot = new TaskStateSnapshot(); final AtomicBoolean jmReported = new AtomicBoolean(false); final AtomicBoolean tmReported = new AtomicBoolean(false); TestCheckpointResponder checkpointResponder = new TestCheckpointResponder() { @Override public void acknowledgeCheckpoint( JobID lJobID, ExecutionAttemptID lExecutionAttemptID, long lCheckpointId, CheckpointMetrics lCheckpointMetrics, TaskStateSnapshot lSubtaskState) { Assert.assertEquals(jobID, lJobID); Assert.assertEquals(executionAttemptID, lExecutionAttemptID); Assert.assertEquals(checkpointMetaData.getCheckpointId(), lCheckpointId); Assert.assertEquals(checkpointMetrics, lCheckpointMetrics); jmReported.set(true); } }; Executor executor = Executors.directExecutor(); LocalRecoveryDirectoryProviderImpl directoryProvider = new LocalRecoveryDirectoryProviderImpl( temporaryFolder.newFolder(), jobID, jobVertexID, subtaskIdx); LocalRecoveryConfig localRecoveryConfig = new LocalRecoveryConfig(true, directoryProvider); TaskLocalStateStore taskLocalStateStore = new TaskLocalStateStoreImpl(jobID, allocationID, jobVertexID, subtaskIdx, localRecoveryConfig, executor) { @Override public void storeLocalState( @Nonnegative long checkpointId, @Nullable TaskStateSnapshot localState) { Assert.assertEquals(tmSnapshot, localState); tmReported.set(true); } }; TaskStateManagerImpl taskStateManager = new TaskStateManagerImpl( jobID, executionAttemptID, taskLocalStateStore, null, checkpointResponder); taskStateManager.reportTaskStateSnapshots( checkpointMetaData, checkpointMetrics, jmSnapshot, tmSnapshot); Assert.assertTrue("Reporting for JM state was not called.", jmReported.get()); Assert.assertTrue("Reporting for TM state was not called.", tmReported.get()); }
Example #9
Source File: TaskStateManagerImplTest.java From flink with Apache License 2.0 | 4 votes |
/** * This tests if the {@link TaskStateManager} properly returns the the subtask local state dir from the * corresponding {@link TaskLocalStateStoreImpl}. */ @Test public void testForwardingSubtaskLocalStateBaseDirFromLocalStateStore() throws IOException { JobID jobID = new JobID(42L, 43L); AllocationID allocationID = new AllocationID(4711L, 23L); JobVertexID jobVertexID = new JobVertexID(12L, 34L); ExecutionAttemptID executionAttemptID = new ExecutionAttemptID(23L, 24L); TestCheckpointResponder checkpointResponderMock = new TestCheckpointResponder(); Executor directExecutor = Executors.directExecutor(); TemporaryFolder tmpFolder = new TemporaryFolder(); try { tmpFolder.create(); File[] allocBaseDirs = new File[]{tmpFolder.newFolder(), tmpFolder.newFolder(), tmpFolder.newFolder()}; LocalRecoveryDirectoryProviderImpl directoryProvider = new LocalRecoveryDirectoryProviderImpl(allocBaseDirs, jobID, jobVertexID, 0); LocalRecoveryConfig localRecoveryConfig = new LocalRecoveryConfig(true, directoryProvider); TaskLocalStateStore taskLocalStateStore = new TaskLocalStateStoreImpl(jobID, allocationID, jobVertexID, 13, localRecoveryConfig, directExecutor); TaskStateManager taskStateManager = taskStateManager( jobID, executionAttemptID, checkpointResponderMock, null, taskLocalStateStore); LocalRecoveryConfig localRecoveryConfFromTaskLocalStateStore = taskLocalStateStore.getLocalRecoveryConfig(); LocalRecoveryConfig localRecoveryConfFromTaskStateManager = taskStateManager.createLocalRecoveryConfig(); for (int i = 0; i < 10; ++i) { Assert.assertEquals(allocBaseDirs[i % allocBaseDirs.length], localRecoveryConfFromTaskLocalStateStore.getLocalStateDirectoryProvider().allocationBaseDirectory(i)); Assert.assertEquals(allocBaseDirs[i % allocBaseDirs.length], localRecoveryConfFromTaskStateManager.getLocalStateDirectoryProvider().allocationBaseDirectory(i)); } Assert.assertEquals( localRecoveryConfFromTaskLocalStateStore.isLocalRecoveryEnabled(), localRecoveryConfFromTaskStateManager.isLocalRecoveryEnabled()); } finally { tmpFolder.delete(); } }
Example #10
Source File: StreamTaskStateInitializerImplTest.java From flink with Apache License 2.0 | 4 votes |
private StreamTaskStateInitializer streamTaskStateManager( StateBackend stateBackend, JobManagerTaskRestore jobManagerTaskRestore, boolean createTimerServiceManager) { JobID jobID = new JobID(42L, 43L); ExecutionAttemptID executionAttemptID = new ExecutionAttemptID(23L, 24L); TestCheckpointResponder checkpointResponderMock = new TestCheckpointResponder(); TaskLocalStateStore taskLocalStateStore = new TestTaskLocalStateStore(); TaskStateManager taskStateManager = TaskStateManagerImplTest.taskStateManager( jobID, executionAttemptID, checkpointResponderMock, jobManagerTaskRestore, taskLocalStateStore); DummyEnvironment dummyEnvironment = new DummyEnvironment("test-task", 1, 0); dummyEnvironment.setTaskStateManager(taskStateManager); ProcessingTimeService processingTimeService = new TestProcessingTimeService(); if (createTimerServiceManager) { return new StreamTaskStateInitializerImpl( dummyEnvironment, stateBackend, processingTimeService); } else { return new StreamTaskStateInitializerImpl( dummyEnvironment, stateBackend, processingTimeService) { @Override protected <K> InternalTimeServiceManager<K> internalTimeServiceManager( AbstractKeyedStateBackend<K> keyedStatedBackend, KeyContext keyContext, Iterable<KeyGroupStatePartitionStreamProvider> rawKeyedStates) throws Exception { return null; } }; } }
Example #11
Source File: LocalStateForwardingTest.java From flink with Apache License 2.0 | 4 votes |
/** * This tests that state that was reported to the {@link org.apache.flink.runtime.state.TaskStateManager} is also * reported to {@link org.apache.flink.runtime.taskmanager.CheckpointResponder} and {@link TaskLocalStateStoreImpl}. */ @Test public void testReportingFromTaskStateManagerToResponderAndTaskLocalStateStore() throws Exception { final JobID jobID = new JobID(); final AllocationID allocationID = new AllocationID(); final ExecutionAttemptID executionAttemptID = new ExecutionAttemptID(); final CheckpointMetaData checkpointMetaData = new CheckpointMetaData(42L, 4711L); final CheckpointMetrics checkpointMetrics = new CheckpointMetrics(); final int subtaskIdx = 42; JobVertexID jobVertexID = new JobVertexID(); TaskStateSnapshot jmSnapshot = new TaskStateSnapshot(); TaskStateSnapshot tmSnapshot = new TaskStateSnapshot(); final AtomicBoolean jmReported = new AtomicBoolean(false); final AtomicBoolean tmReported = new AtomicBoolean(false); TestCheckpointResponder checkpointResponder = new TestCheckpointResponder() { @Override public void acknowledgeCheckpoint( JobID lJobID, ExecutionAttemptID lExecutionAttemptID, long lCheckpointId, CheckpointMetrics lCheckpointMetrics, TaskStateSnapshot lSubtaskState) { Assert.assertEquals(jobID, lJobID); Assert.assertEquals(executionAttemptID, lExecutionAttemptID); Assert.assertEquals(checkpointMetaData.getCheckpointId(), lCheckpointId); Assert.assertEquals(checkpointMetrics, lCheckpointMetrics); jmReported.set(true); } }; Executor executor = Executors.directExecutor(); LocalRecoveryDirectoryProviderImpl directoryProvider = new LocalRecoveryDirectoryProviderImpl( temporaryFolder.newFolder(), jobID, jobVertexID, subtaskIdx); LocalRecoveryConfig localRecoveryConfig = new LocalRecoveryConfig(true, directoryProvider); TaskLocalStateStore taskLocalStateStore = new TaskLocalStateStoreImpl(jobID, allocationID, jobVertexID, subtaskIdx, localRecoveryConfig, executor) { @Override public void storeLocalState( @Nonnegative long checkpointId, @Nullable TaskStateSnapshot localState) { Assert.assertEquals(tmSnapshot, localState); tmReported.set(true); } }; TaskStateManagerImpl taskStateManager = new TaskStateManagerImpl( jobID, executionAttemptID, taskLocalStateStore, null, checkpointResponder); taskStateManager.reportTaskStateSnapshots( checkpointMetaData, checkpointMetrics, jmSnapshot, tmSnapshot); Assert.assertTrue("Reporting for JM state was not called.", jmReported.get()); Assert.assertTrue("Reporting for TM state was not called.", tmReported.get()); }
Example #12
Source File: TaskStateManagerImplTest.java From flink with Apache License 2.0 | 4 votes |
/** * This tests if the {@link TaskStateManager} properly returns the the subtask local state dir from the * corresponding {@link TaskLocalStateStoreImpl}. */ @Test public void testForwardingSubtaskLocalStateBaseDirFromLocalStateStore() throws IOException { JobID jobID = new JobID(42L, 43L); AllocationID allocationID = new AllocationID(4711L, 23L); JobVertexID jobVertexID = new JobVertexID(12L, 34L); ExecutionAttemptID executionAttemptID = new ExecutionAttemptID(23L, 24L); TestCheckpointResponder checkpointResponderMock = new TestCheckpointResponder(); Executor directExecutor = Executors.directExecutor(); TemporaryFolder tmpFolder = new TemporaryFolder(); try { tmpFolder.create(); File[] allocBaseDirs = new File[]{tmpFolder.newFolder(), tmpFolder.newFolder(), tmpFolder.newFolder()}; LocalRecoveryDirectoryProviderImpl directoryProvider = new LocalRecoveryDirectoryProviderImpl(allocBaseDirs, jobID, jobVertexID, 0); LocalRecoveryConfig localRecoveryConfig = new LocalRecoveryConfig(true, directoryProvider); TaskLocalStateStore taskLocalStateStore = new TaskLocalStateStoreImpl(jobID, allocationID, jobVertexID, 13, localRecoveryConfig, directExecutor); TaskStateManager taskStateManager = taskStateManager( jobID, executionAttemptID, checkpointResponderMock, null, taskLocalStateStore); LocalRecoveryConfig localRecoveryConfFromTaskLocalStateStore = taskLocalStateStore.getLocalRecoveryConfig(); LocalRecoveryConfig localRecoveryConfFromTaskStateManager = taskStateManager.createLocalRecoveryConfig(); for (int i = 0; i < 10; ++i) { Assert.assertEquals(allocBaseDirs[i % allocBaseDirs.length], localRecoveryConfFromTaskLocalStateStore.getLocalStateDirectoryProvider().allocationBaseDirectory(i)); Assert.assertEquals(allocBaseDirs[i % allocBaseDirs.length], localRecoveryConfFromTaskStateManager.getLocalStateDirectoryProvider().allocationBaseDirectory(i)); } Assert.assertEquals( localRecoveryConfFromTaskLocalStateStore.isLocalRecoveryEnabled(), localRecoveryConfFromTaskStateManager.isLocalRecoveryEnabled()); } finally { tmpFolder.delete(); } }
Example #13
Source File: LocalStateForwardingTest.java From flink with Apache License 2.0 | 4 votes |
/** * This tests that state that was reported to the {@link org.apache.flink.runtime.state.TaskStateManager} is also * reported to {@link org.apache.flink.runtime.taskmanager.CheckpointResponder} and {@link TaskLocalStateStoreImpl}. */ @Test public void testReportingFromTaskStateManagerToResponderAndTaskLocalStateStore() throws Exception { final JobID jobID = new JobID(); final AllocationID allocationID = new AllocationID(); final ExecutionAttemptID executionAttemptID = new ExecutionAttemptID(); final CheckpointMetaData checkpointMetaData = new CheckpointMetaData(42L, 4711L); final CheckpointMetrics checkpointMetrics = new CheckpointMetrics(); final int subtaskIdx = 42; JobVertexID jobVertexID = new JobVertexID(); TaskStateSnapshot jmSnapshot = new TaskStateSnapshot(); TaskStateSnapshot tmSnapshot = new TaskStateSnapshot(); final AtomicBoolean jmReported = new AtomicBoolean(false); final AtomicBoolean tmReported = new AtomicBoolean(false); TestCheckpointResponder checkpointResponder = new TestCheckpointResponder() { @Override public void acknowledgeCheckpoint( JobID lJobID, ExecutionAttemptID lExecutionAttemptID, long lCheckpointId, CheckpointMetrics lCheckpointMetrics, TaskStateSnapshot lSubtaskState) { Assert.assertEquals(jobID, lJobID); Assert.assertEquals(executionAttemptID, lExecutionAttemptID); Assert.assertEquals(checkpointMetaData.getCheckpointId(), lCheckpointId); Assert.assertEquals(checkpointMetrics, lCheckpointMetrics); jmReported.set(true); } }; Executor executor = Executors.directExecutor(); LocalRecoveryDirectoryProviderImpl directoryProvider = new LocalRecoveryDirectoryProviderImpl( temporaryFolder.newFolder(), jobID, jobVertexID, subtaskIdx); LocalRecoveryConfig localRecoveryConfig = new LocalRecoveryConfig(true, directoryProvider); TaskLocalStateStore taskLocalStateStore = new TaskLocalStateStoreImpl(jobID, allocationID, jobVertexID, subtaskIdx, localRecoveryConfig, executor) { @Override public void storeLocalState( @Nonnegative long checkpointId, @Nullable TaskStateSnapshot localState) { Assert.assertEquals(tmSnapshot, localState); tmReported.set(true); } }; TaskStateManagerImpl taskStateManager = new TaskStateManagerImpl( jobID, executionAttemptID, taskLocalStateStore, null, checkpointResponder); taskStateManager.reportTaskStateSnapshots( checkpointMetaData, checkpointMetrics, jmSnapshot, tmSnapshot); Assert.assertTrue("Reporting for JM state was not called.", jmReported.get()); Assert.assertTrue("Reporting for TM state was not called.", tmReported.get()); }
Example #14
Source File: StreamTaskMailboxTestHarness.java From flink with Apache License 2.0 | 4 votes |
public TestCheckpointResponder getCheckpointResponder() { return (TestCheckpointResponder) taskStateManager.getCheckpointResponder(); }