Java Code Examples for org.apache.flink.runtime.checkpoint.TaskStateSnapshot#putSubtaskStateByOperatorID()

The following examples show how to use org.apache.flink.runtime.checkpoint.TaskStateSnapshot#putSubtaskStateByOperatorID() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: JobMasterTriggerSavepointIT.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Override
public boolean triggerCheckpoint(final CheckpointMetaData checkpointMetaData, final CheckpointOptions checkpointOptions) {
	final TaskStateSnapshot checkpointStateHandles = new TaskStateSnapshot();
	checkpointStateHandles.putSubtaskStateByOperatorID(
		OperatorID.fromJobVertexID(getEnvironment().getJobVertexId()),
		new OperatorSubtaskState());

	getEnvironment().acknowledgeCheckpoint(
		checkpointMetaData.getCheckpointId(),
		new CheckpointMetrics(),
		checkpointStateHandles);

	triggerCheckpointLatch.countDown();

	return true;
}
Example 2
Source File: JobMasterTriggerSavepointITCase.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
public boolean triggerCheckpoint(final CheckpointMetaData checkpointMetaData, final CheckpointOptions checkpointOptions, final boolean advanceToEndOfEventTime) {
	final TaskStateSnapshot checkpointStateHandles = new TaskStateSnapshot();
	checkpointStateHandles.putSubtaskStateByOperatorID(
		OperatorID.fromJobVertexID(getEnvironment().getJobVertexId()),
		new OperatorSubtaskState());

	getEnvironment().acknowledgeCheckpoint(
		checkpointMetaData.getCheckpointId(),
		new CheckpointMetrics(),
		checkpointStateHandles);

	triggerCheckpointLatch.countDown();

	return true;
}
Example 3
Source File: FailoverRegionTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Let the checkpoint coordinator to receive all acknowledges from given executionVertexes so that to complete the expected checkpoint.
 */
private void acknowledgeAllCheckpoints(CheckpointCoordinator checkpointCoordinator, Iterator<ExecutionVertex> executionVertexes) throws IOException, CheckpointException {
	while (executionVertexes.hasNext()) {
		ExecutionVertex executionVertex = executionVertexes.next();
		for (int index = 0; index < executionVertex.getJobVertex().getParallelism(); index++) {
			JobVertexID jobVertexID = executionVertex.getJobvertexId();
			OperatorStateHandle opStateBackend = CheckpointCoordinatorTest.generatePartitionableStateHandle(jobVertexID, index, 2, 8, false);
			OperatorSubtaskState operatorSubtaskState = new OperatorSubtaskState(opStateBackend, null, null, null);
			TaskStateSnapshot taskOperatorSubtaskStates = new TaskStateSnapshot();
			taskOperatorSubtaskStates.putSubtaskStateByOperatorID(OperatorID.fromJobVertexID(jobVertexID), operatorSubtaskState);

			AcknowledgeCheckpoint acknowledgeCheckpoint = new AcknowledgeCheckpoint(
				executionVertex.getJobId(),
				executionVertex.getJobVertex().getTaskVertices()[index].getCurrentExecutionAttempt().getAttemptId(),
				checkpointId,
				new CheckpointMetrics(),
				taskOperatorSubtaskStates);

			checkpointCoordinator.receiveAcknowledgeMessage(acknowledgeCheckpoint, "Unknown location");
		}
	}
}
Example 4
Source File: JobMasterTriggerSavepointITCase.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
public Future<Boolean> triggerCheckpointAsync(final CheckpointMetaData checkpointMetaData, final CheckpointOptions checkpointOptions, final boolean advanceToEndOfEventTime) {
	final TaskStateSnapshot checkpointStateHandles = new TaskStateSnapshot();
	checkpointStateHandles.putSubtaskStateByOperatorID(
		OperatorID.fromJobVertexID(getEnvironment().getJobVertexId()),
		new OperatorSubtaskState());

	getEnvironment().acknowledgeCheckpoint(
		checkpointMetaData.getCheckpointId(),
		new CheckpointMetrics(),
		checkpointStateHandles);

	triggerCheckpointLatch.countDown();

	return CompletableFuture.completedFuture(true);
}
Example 5
Source File: CheckpointMessagesTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Test
public void testConfirmTaskCheckpointed() {
	try {
		AcknowledgeCheckpoint noState = new AcknowledgeCheckpoint(
				new JobID(), new ExecutionAttemptID(), 569345L);

		KeyGroupRange keyGroupRange = KeyGroupRange.of(42,42);

		TaskStateSnapshot checkpointStateHandles = new TaskStateSnapshot();
		checkpointStateHandles.putSubtaskStateByOperatorID(
			new OperatorID(),
			new OperatorSubtaskState(
				CheckpointCoordinatorTest.generatePartitionableStateHandle(new JobVertexID(), 0, 2, 8, false),
				null,
				CheckpointCoordinatorTest.generateKeyGroupState(keyGroupRange, Collections.singletonList(new MyHandle())),
				null
			)
		);

		AcknowledgeCheckpoint withState = new AcknowledgeCheckpoint(
				new JobID(),
				new ExecutionAttemptID(),
				87658976143L,
				new CheckpointMetrics(),
				checkpointStateHandles);

		testSerializabilityEqualsHashCode(noState);
		testSerializabilityEqualsHashCode(withState);
	} catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
Example 6
Source File: TaskLocalStateStoreImplTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
private List<TaskStateSnapshot> storeStates(int count) {
	List<TaskStateSnapshot> taskStateSnapshots = new ArrayList<>(count);
	for (int i = 0; i < count; ++i) {
		OperatorID operatorID = new OperatorID();
		TaskStateSnapshot taskStateSnapshot = spy(new TaskStateSnapshot());
		OperatorSubtaskState operatorSubtaskState = new OperatorSubtaskState();
		taskStateSnapshot.putSubtaskStateByOperatorID(operatorID, operatorSubtaskState);
		taskLocalStateStore.storeLocalState(i, taskStateSnapshot);
		taskStateSnapshots.add(taskStateSnapshot);
	}
	return taskStateSnapshots;
}
Example 7
Source File: RestoreStreamTaskTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Test
public void testRestoreAfterScaleUp() throws Exception {
	OperatorID headOperatorID = new OperatorID(42L, 42L);
	OperatorID tailOperatorID = new OperatorID(44L, 44L);

	JobManagerTaskRestore restore = createRunAndCheckpointOperatorChain(
		headOperatorID,
		new CounterOperator(),
		tailOperatorID,
		new CounterOperator(),
		Optional.empty());

	TaskStateSnapshot stateHandles = restore.getTaskStateSnapshot();

	assertEquals(2, stateHandles.getSubtaskStateMappings().size());

	// test empty state in case of scale up
	OperatorSubtaskState emptyHeadOperatorState = StateAssignmentOperation.operatorSubtaskStateFrom(
		new OperatorInstanceID(0, headOperatorID),
		Collections.emptyMap(),
		Collections.emptyMap(),
		Collections.emptyMap(),
		Collections.emptyMap());

	stateHandles.putSubtaskStateByOperatorID(headOperatorID, emptyHeadOperatorState);

	createRunAndCheckpointOperatorChain(
		headOperatorID,
		new CounterOperator(),
		tailOperatorID,
		new CounterOperator(),
		Optional.of(restore));

	assertEquals(new HashSet<>(Arrays.asList(headOperatorID, tailOperatorID)), RESTORED_OPERATORS);
}
Example 8
Source File: AbstractStreamOperatorTestHarness.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
/**
 * Calls {@link org.apache.flink.streaming.api.operators.StreamOperator#initializeState()}.
 * Calls {@link org.apache.flink.streaming.api.operators.StreamOperator#setup(StreamTask, StreamConfig, Output)}
 * if it was not called before.
 *
 * @param jmOperatorStateHandles the primary state (owned by JM)
 * @param tmOperatorStateHandles the (optional) local state (owned by TM) or null.
 * @throws Exception
 */
public void initializeState(
	OperatorSubtaskState jmOperatorStateHandles,
	OperatorSubtaskState tmOperatorStateHandles) throws Exception {

	checkState(!initializeCalled, "TestHarness has already been initialized. Have you " +
		"opened this harness before initializing it?");
	if (!setupCalled) {
		setup();
	}

	if (jmOperatorStateHandles != null) {

		TaskStateSnapshot jmTaskStateSnapshot = new TaskStateSnapshot();
		jmTaskStateSnapshot.putSubtaskStateByOperatorID(operator.getOperatorID(), jmOperatorStateHandles);

		taskStateManager.setReportedCheckpointId(0);
		taskStateManager.setJobManagerTaskStateSnapshotsByCheckpointId(
			Collections.singletonMap(0L, jmTaskStateSnapshot));

		if (tmOperatorStateHandles != null) {
			TaskStateSnapshot tmTaskStateSnapshot = new TaskStateSnapshot();
			tmTaskStateSnapshot.putSubtaskStateByOperatorID(operator.getOperatorID(), tmOperatorStateHandles);
			taskStateManager.setTaskManagerTaskStateSnapshotsByCheckpointId(
				Collections.singletonMap(0L, tmTaskStateSnapshot));
		}
	}

	operator.initializeState();
	initializeCalled = true;
}
Example 9
Source File: CheckpointMessagesTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testConfirmTaskCheckpointed() {
	try {
		AcknowledgeCheckpoint noState = new AcknowledgeCheckpoint(
				new JobID(), new ExecutionAttemptID(), 569345L);

		KeyGroupRange keyGroupRange = KeyGroupRange.of(42,42);

		TaskStateSnapshot checkpointStateHandles = new TaskStateSnapshot();
		checkpointStateHandles.putSubtaskStateByOperatorID(
			new OperatorID(),
			new OperatorSubtaskState(
				CheckpointCoordinatorTest.generatePartitionableStateHandle(new JobVertexID(), 0, 2, 8, false),
				null,
				CheckpointCoordinatorTest.generateKeyGroupState(keyGroupRange, Collections.singletonList(new MyHandle())),
				null
			)
		);

		AcknowledgeCheckpoint withState = new AcknowledgeCheckpoint(
				new JobID(),
				new ExecutionAttemptID(),
				87658976143L,
				new CheckpointMetrics(),
				checkpointStateHandles);

		testSerializabilityEqualsHashCode(noState);
		testSerializabilityEqualsHashCode(withState);
	} catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
Example 10
Source File: TaskLocalStateStoreImplTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
private List<TaskStateSnapshot> storeStates(int count) {
	List<TaskStateSnapshot> taskStateSnapshots = new ArrayList<>(count);
	for (int i = 0; i < count; ++i) {
		OperatorID operatorID = new OperatorID();
		TaskStateSnapshot taskStateSnapshot = spy(new TaskStateSnapshot());
		OperatorSubtaskState operatorSubtaskState = new OperatorSubtaskState();
		taskStateSnapshot.putSubtaskStateByOperatorID(operatorID, operatorSubtaskState);
		taskLocalStateStore.storeLocalState(i, taskStateSnapshot);
		taskStateSnapshots.add(taskStateSnapshot);
	}
	return taskStateSnapshots;
}
Example 11
Source File: RestoreStreamTaskTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testRestoreAfterScaleUp() throws Exception {
	OperatorID headOperatorID = new OperatorID(42L, 42L);
	OperatorID tailOperatorID = new OperatorID(44L, 44L);

	JobManagerTaskRestore restore = createRunAndCheckpointOperatorChain(
		headOperatorID,
		new CounterOperator(),
		tailOperatorID,
		new CounterOperator(),
		Optional.empty());

	TaskStateSnapshot stateHandles = restore.getTaskStateSnapshot();

	assertEquals(2, stateHandles.getSubtaskStateMappings().size());

	// test empty state in case of scale up
	OperatorSubtaskState emptyHeadOperatorState = StateAssignmentOperation.operatorSubtaskStateFrom(
		new OperatorInstanceID(0, headOperatorID),
		Collections.emptyMap(),
		Collections.emptyMap(),
		Collections.emptyMap(),
		Collections.emptyMap());

	stateHandles.putSubtaskStateByOperatorID(headOperatorID, emptyHeadOperatorState);

	createRunAndCheckpointOperatorChain(
		headOperatorID,
		new CounterOperator(),
		tailOperatorID,
		new CounterOperator(),
		Optional.of(restore));

	assertEquals(new HashSet<>(Arrays.asList(headOperatorID, tailOperatorID)), RESTORED_OPERATORS);
}
Example 12
Source File: AbstractStreamOperatorTestHarness.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Calls {@link org.apache.flink.streaming.api.operators.StreamOperator#initializeState()}.
 * Calls {@link org.apache.flink.streaming.api.operators.SetupableStreamOperator#setup(StreamTask, StreamConfig, Output)}
 * if it was not called before.
 *
 * @param jmOperatorStateHandles the primary state (owned by JM)
 * @param tmOperatorStateHandles the (optional) local state (owned by TM) or null.
 * @throws Exception
 */
public void initializeState(
	OperatorSubtaskState jmOperatorStateHandles,
	OperatorSubtaskState tmOperatorStateHandles) throws Exception {

	checkState(!initializeCalled, "TestHarness has already been initialized. Have you " +
		"opened this harness before initializing it?");
	if (!setupCalled) {
		setup();
	}

	if (jmOperatorStateHandles != null) {

		TaskStateSnapshot jmTaskStateSnapshot = new TaskStateSnapshot();
		jmTaskStateSnapshot.putSubtaskStateByOperatorID(operator.getOperatorID(), jmOperatorStateHandles);

		taskStateManager.setReportedCheckpointId(0);
		taskStateManager.setJobManagerTaskStateSnapshotsByCheckpointId(
			Collections.singletonMap(0L, jmTaskStateSnapshot));

		if (tmOperatorStateHandles != null) {
			TaskStateSnapshot tmTaskStateSnapshot = new TaskStateSnapshot();
			tmTaskStateSnapshot.putSubtaskStateByOperatorID(operator.getOperatorID(), tmOperatorStateHandles);
			taskStateManager.setTaskManagerTaskStateSnapshotsByCheckpointId(
				Collections.singletonMap(0L, tmTaskStateSnapshot));
		}
	}

	operator.initializeState();
	initializeCalled = true;
}
Example 13
Source File: CheckpointMessagesTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testConfirmTaskCheckpointed() {
	final Random rnd = new Random();
	try {
		AcknowledgeCheckpoint noState = new AcknowledgeCheckpoint(
				new JobID(), new ExecutionAttemptID(), 569345L);

		KeyGroupRange keyGroupRange = KeyGroupRange.of(42, 42);

		TaskStateSnapshot checkpointStateHandles = new TaskStateSnapshot();
		checkpointStateHandles.putSubtaskStateByOperatorID(
			new OperatorID(),
			new OperatorSubtaskState(
				CheckpointCoordinatorTestingUtils.generatePartitionableStateHandle(new JobVertexID(), 0, 2, 8, false),
				null,
				CheckpointCoordinatorTestingUtils.generateKeyGroupState(keyGroupRange, Collections.singletonList(new MyHandle())),
				null,
				singleton(createNewInputChannelStateHandle(10, rnd)),
				singleton(createNewResultSubpartitionStateHandle(10, rnd))
			)
		);

		AcknowledgeCheckpoint withState = new AcknowledgeCheckpoint(
				new JobID(),
				new ExecutionAttemptID(),
				87658976143L,
				new CheckpointMetrics(),
				checkpointStateHandles);

		testSerializabilityEqualsHashCode(noState);
		testSerializabilityEqualsHashCode(withState);
	} catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
Example 14
Source File: TaskLocalStateStoreImplTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
private List<TaskStateSnapshot> storeStates(int count) {
	List<TaskStateSnapshot> taskStateSnapshots = new ArrayList<>(count);
	for (int i = 0; i < count; ++i) {
		OperatorID operatorID = new OperatorID();
		TaskStateSnapshot taskStateSnapshot = spy(new TaskStateSnapshot());
		OperatorSubtaskState operatorSubtaskState = new OperatorSubtaskState();
		taskStateSnapshot.putSubtaskStateByOperatorID(operatorID, operatorSubtaskState);
		taskLocalStateStore.storeLocalState(i, taskStateSnapshot);
		taskStateSnapshots.add(taskStateSnapshot);
	}
	return taskStateSnapshots;
}
Example 15
Source File: RestoreStreamTaskTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testRestoreAfterScaleUp() throws Exception {
	OperatorID headOperatorID = new OperatorID(42L, 42L);
	OperatorID tailOperatorID = new OperatorID(44L, 44L);

	JobManagerTaskRestore restore = createRunAndCheckpointOperatorChain(
		headOperatorID,
		new CounterOperator(),
		tailOperatorID,
		new CounterOperator(),
		Optional.empty());

	TaskStateSnapshot stateHandles = restore.getTaskStateSnapshot();

	assertEquals(2, stateHandles.getSubtaskStateMappings().size());

	// test empty state in case of scale up
	OperatorSubtaskState emptyHeadOperatorState = StateAssignmentOperation.operatorSubtaskStateFrom(
		new OperatorInstanceID(0, headOperatorID),
		Collections.emptyMap(),
		Collections.emptyMap(),
		Collections.emptyMap(),
		Collections.emptyMap(),
		Collections.emptyMap(),
		Collections.emptyMap());

	stateHandles.putSubtaskStateByOperatorID(headOperatorID, emptyHeadOperatorState);

	createRunAndCheckpointOperatorChain(
		headOperatorID,
		new CounterOperator(),
		tailOperatorID,
		new CounterOperator(),
		Optional.of(restore));

	assertEquals(new HashSet<>(Arrays.asList(headOperatorID, tailOperatorID)), RESTORED_OPERATORS);
}
Example 16
Source File: AbstractStreamOperatorTestHarness.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Calls {@link org.apache.flink.streaming.api.operators.StreamOperator#initializeState()}.
 * Calls {@link org.apache.flink.streaming.api.operators.SetupableStreamOperator#setup(StreamTask, StreamConfig, Output)}
 * if it was not called before.
 *
 * @param jmOperatorStateHandles the primary state (owned by JM)
 * @param tmOperatorStateHandles the (optional) local state (owned by TM) or null.
 * @throws Exception
 */
public void initializeState(
	OperatorSubtaskState jmOperatorStateHandles,
	OperatorSubtaskState tmOperatorStateHandles) throws Exception {

	checkState(!initializeCalled, "TestHarness has already been initialized. Have you " +
		"opened this harness before initializing it?");
	if (!setupCalled) {
		setup();
	}

	if (jmOperatorStateHandles != null) {

		TaskStateSnapshot jmTaskStateSnapshot = new TaskStateSnapshot();
		jmTaskStateSnapshot.putSubtaskStateByOperatorID(operator.getOperatorID(), jmOperatorStateHandles);

		taskStateManager.setReportedCheckpointId(0);
		taskStateManager.setJobManagerTaskStateSnapshotsByCheckpointId(
			Collections.singletonMap(0L, jmTaskStateSnapshot));

		if (tmOperatorStateHandles != null) {
			TaskStateSnapshot tmTaskStateSnapshot = new TaskStateSnapshot();
			tmTaskStateSnapshot.putSubtaskStateByOperatorID(operator.getOperatorID(), tmOperatorStateHandles);
			taskStateManager.setTaskManagerTaskStateSnapshotsByCheckpointId(
				Collections.singletonMap(0L, tmTaskStateSnapshot));
		}
	}

	operator.initializeState(mockTask.createStreamTaskStateInitializer());
	initializeCalled = true;
}