Java Code Examples for org.apache.flink.runtime.executiongraph.ExecutionJobVertex#getParallelism()
The following examples show how to use
org.apache.flink.runtime.executiongraph.ExecutionJobVertex#getParallelism() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: CheckpointCoordinatorTestingUtils.java From flink with Apache License 2.0 | 6 votes |
|
public static void verifyStateRestore(
JobVertexID jobVertexID, ExecutionJobVertex executionJobVertex,
List<KeyGroupRange> keyGroupPartitions) throws Exception {
for (int i = 0; i < executionJobVertex.getParallelism(); i++) {
JobManagerTaskRestore taskRestore = executionJobVertex.getTaskVertices()[i].getCurrentExecutionAttempt().getTaskRestore();
Assert.assertEquals(1L, taskRestore.getRestoreCheckpointId());
TaskStateSnapshot stateSnapshot = taskRestore.getTaskStateSnapshot();
OperatorSubtaskState operatorState = stateSnapshot.getSubtaskStateByOperatorID(OperatorID.fromJobVertexID(jobVertexID));
ChainedStateHandle<OperatorStateHandle> expectedOpStateBackend =
generateChainedPartitionableStateHandle(jobVertexID, i, 2, 8, false);
assertTrue(CommonTestUtils.isStreamContentEqual(
expectedOpStateBackend.get(0).openInputStream(),
operatorState.getManagedOperatorState().iterator().next().openInputStream()));
KeyGroupsStateHandle expectPartitionedKeyGroupState = generateKeyGroupState(
jobVertexID, keyGroupPartitions.get(i), false);
compareKeyedState(Collections.singletonList(expectPartitionedKeyGroupState), operatorState.getManagedKeyedState());
}
}
Example 2
| Source File: CheckpointCoordinatorTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
|
public static void verifyStateRestore(
JobVertexID jobVertexID, ExecutionJobVertex executionJobVertex,
List<KeyGroupRange> keyGroupPartitions) throws Exception {
for (int i = 0; i < executionJobVertex.getParallelism(); i++) {
JobManagerTaskRestore taskRestore = executionJobVertex.getTaskVertices()[i].getCurrentExecutionAttempt().getTaskRestore();
Assert.assertEquals(1L, taskRestore.getRestoreCheckpointId());
TaskStateSnapshot stateSnapshot = taskRestore.getTaskStateSnapshot();
OperatorSubtaskState operatorState = stateSnapshot.getSubtaskStateByOperatorID(OperatorID.fromJobVertexID(jobVertexID));
ChainedStateHandle<OperatorStateHandle> expectedOpStateBackend =
generateChainedPartitionableStateHandle(jobVertexID, i, 2, 8, false);
assertTrue(CommonTestUtils.isSteamContentEqual(
expectedOpStateBackend.get(0).openInputStream(),
operatorState.getManagedOperatorState().iterator().next().openInputStream()));
KeyGroupsStateHandle expectPartitionedKeyGroupState = generateKeyGroupState(
jobVertexID, keyGroupPartitions.get(i), false);
compareKeyedState(Collections.singletonList(expectPartitionedKeyGroupState), operatorState.getManagedKeyedState());
}
}
Example 3
| Source File: CheckpointCoordinatorTest.java From flink with Apache License 2.0 | 6 votes |
|
public static void verifyStateRestore(
JobVertexID jobVertexID, ExecutionJobVertex executionJobVertex,
List<KeyGroupRange> keyGroupPartitions) throws Exception {
for (int i = 0; i < executionJobVertex.getParallelism(); i++) {
JobManagerTaskRestore taskRestore = executionJobVertex.getTaskVertices()[i].getCurrentExecutionAttempt().getTaskRestore();
Assert.assertEquals(1L, taskRestore.getRestoreCheckpointId());
TaskStateSnapshot stateSnapshot = taskRestore.getTaskStateSnapshot();
OperatorSubtaskState operatorState = stateSnapshot.getSubtaskStateByOperatorID(OperatorID.fromJobVertexID(jobVertexID));
ChainedStateHandle<OperatorStateHandle> expectedOpStateBackend =
generateChainedPartitionableStateHandle(jobVertexID, i, 2, 8, false);
assertTrue(CommonTestUtils.isStreamContentEqual(
expectedOpStateBackend.get(0).openInputStream(),
operatorState.getManagedOperatorState().iterator().next().openInputStream()));
KeyGroupsStateHandle expectPartitionedKeyGroupState = generateKeyGroupState(
jobVertexID, keyGroupPartitions.get(i), false);
compareKeyedState(Collections.singletonList(expectPartitionedKeyGroupState), operatorState.getManagedKeyedState());
}
}
Example 4
| Source File: StateAssignmentOperation.java From flink with Apache License 2.0 | 5 votes |
|
/**
* Verifies conditions in regards to parallelism and maxParallelism that must be met when restoring state.
*
* @param operatorState state to restore
* @param executionJobVertex task for which the state should be restored
*/
private static void checkParallelismPreconditions(OperatorState operatorState, ExecutionJobVertex executionJobVertex) {
//----------------------------------------max parallelism preconditions-------------------------------------
if (operatorState.getMaxParallelism() < executionJobVertex.getParallelism()) {
throw new IllegalStateException("The state for task " + executionJobVertex.getJobVertexId() +
" can not be restored. The maximum parallelism (" + operatorState.getMaxParallelism() +
") of the restored state is lower than the configured parallelism (" + executionJobVertex.getParallelism() +
"). Please reduce the parallelism of the task to be lower or equal to the maximum parallelism."
);
}
// check that the number of key groups have not changed or if we need to override it to satisfy the restored state
if (operatorState.getMaxParallelism() != executionJobVertex.getMaxParallelism()) {
if (!executionJobVertex.isMaxParallelismConfigured()) {
// if the max parallelism was not explicitly specified by the user, we derive it from the state
LOG.debug("Overriding maximum parallelism for JobVertex {} from {} to {}",
executionJobVertex.getJobVertexId(), executionJobVertex.getMaxParallelism(), operatorState.getMaxParallelism());
executionJobVertex.setMaxParallelism(operatorState.getMaxParallelism());
} else {
// if the max parallelism was explicitly specified, we complain on mismatch
throw new IllegalStateException("The maximum parallelism (" +
operatorState.getMaxParallelism() + ") with which the latest " +
"checkpoint of the execution job vertex " + executionJobVertex +
" has been taken and the current maximum parallelism (" +
executionJobVertex.getMaxParallelism() + ") changed. This " +
"is currently not supported.");
}
}
}
Example 5
| Source File: StateAssignmentOperation.java From flink with Apache License 2.0 | 5 votes |
|
/**
* Verifies conditions in regards to parallelism and maxParallelism that must be met when restoring state.
*
* @param operatorState state to restore
* @param executionJobVertex task for which the state should be restored
*/
private static void checkParallelismPreconditions(OperatorState operatorState, ExecutionJobVertex executionJobVertex) {
//----------------------------------------max parallelism preconditions-------------------------------------
if (operatorState.getMaxParallelism() < executionJobVertex.getParallelism()) {
throw new IllegalStateException("The state for task " + executionJobVertex.getJobVertexId() +
" can not be restored. The maximum parallelism (" + operatorState.getMaxParallelism() +
") of the restored state is lower than the configured parallelism (" + executionJobVertex.getParallelism() +
"). Please reduce the parallelism of the task to be lower or equal to the maximum parallelism."
);
}
// check that the number of key groups have not changed or if we need to override it to satisfy the restored state
if (operatorState.getMaxParallelism() != executionJobVertex.getMaxParallelism()) {
if (!executionJobVertex.isMaxParallelismConfigured()) {
// if the max parallelism was not explicitly specified by the user, we derive it from the state
LOG.debug("Overriding maximum parallelism for JobVertex {} from {} to {}",
executionJobVertex.getJobVertexId(), executionJobVertex.getMaxParallelism(), operatorState.getMaxParallelism());
executionJobVertex.setMaxParallelism(operatorState.getMaxParallelism());
} else {
// if the max parallelism was explicitly specified, we complain on mismatch
throw new IllegalStateException("The maximum parallelism (" +
operatorState.getMaxParallelism() + ") with which the latest " +
"checkpoint of the execution job vertex " + executionJobVertex +
" has been taken and the current maximum parallelism (" +
executionJobVertex.getMaxParallelism() + ") changed. This " +
"is currently not supported.");
}
}
}
Example 6
| Source File: CheckpointStatsTracker.java From flink with Apache License 2.0 | 5 votes |
|
/**
* Creates an empty map with a {@link TaskStateStats} instance per task
* that is involved in the checkpoint.
*
* @return An empty map with an {@link TaskStateStats} entry for each task that is involved in the checkpoint.
*/
private ConcurrentHashMap<JobVertexID, TaskStateStats> createEmptyTaskStateStatsMap() {
ConcurrentHashMap<JobVertexID, TaskStateStats> taskStatsMap = new ConcurrentHashMap<>(jobVertices.size());
for (ExecutionJobVertex vertex : jobVertices) {
TaskStateStats taskStats = new TaskStateStats(vertex.getJobVertexId(), vertex.getParallelism());
taskStatsMap.put(vertex.getJobVertexId(), taskStats);
}
return taskStatsMap;
}
Example 7
| Source File: CheckpointStatsTracker.java From flink with Apache License 2.0 | 5 votes |
|
/**
* Creates a new checkpoint stats tracker.
*
* @param numRememberedCheckpoints Maximum number of checkpoints to remember, including in progress ones.
* @param jobVertices Job vertices involved in the checkpoints.
* @param jobCheckpointingConfiguration Checkpointing configuration.
* @param metricGroup Metric group for exposed metrics
*/
public CheckpointStatsTracker(
int numRememberedCheckpoints,
List<ExecutionJobVertex> jobVertices,
CheckpointCoordinatorConfiguration jobCheckpointingConfiguration,
MetricGroup metricGroup) {
checkArgument(numRememberedCheckpoints >= 0, "Negative number of remembered checkpoints");
this.history = new CheckpointStatsHistory(numRememberedCheckpoints);
this.jobVertices = checkNotNull(jobVertices, "JobVertices");
this.jobCheckpointingConfiguration = checkNotNull(jobCheckpointingConfiguration);
// Compute the total subtask count. We do this here in order to only
// do it once.
int count = 0;
for (ExecutionJobVertex vertex : jobVertices) {
count += vertex.getParallelism();
}
this.totalSubtaskCount = count;
// Latest snapshot is empty
latestSnapshot = new CheckpointStatsSnapshot(
counts.createSnapshot(),
summary.createSnapshot(),
history.createSnapshot(),
null);
// Register the metrics
registerMetrics(metricGroup);
}
Example 8
| Source File: CheckpointStatsTracker.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
/**
* Creates a new checkpoint stats tracker.
*
* @param numRememberedCheckpoints Maximum number of checkpoints to remember, including in progress ones.
* @param jobVertices Job vertices involved in the checkpoints.
* @param jobCheckpointingConfiguration Checkpointing configuration.
* @param metricGroup Metric group for exposed metrics
*/
public CheckpointStatsTracker(
int numRememberedCheckpoints,
List<ExecutionJobVertex> jobVertices,
CheckpointCoordinatorConfiguration jobCheckpointingConfiguration,
MetricGroup metricGroup) {
checkArgument(numRememberedCheckpoints >= 0, "Negative number of remembered checkpoints");
this.history = new CheckpointStatsHistory(numRememberedCheckpoints);
this.jobVertices = checkNotNull(jobVertices, "JobVertices");
this.jobCheckpointingConfiguration = checkNotNull(jobCheckpointingConfiguration);
// Compute the total subtask count. We do this here in order to only
// do it once.
int count = 0;
for (ExecutionJobVertex vertex : jobVertices) {
count += vertex.getParallelism();
}
this.totalSubtaskCount = count;
// Latest snapshot is empty
latestSnapshot = new CheckpointStatsSnapshot(
counts.createSnapshot(),
summary.createSnapshot(),
history.createSnapshot(),
null);
// Register the metrics
registerMetrics(metricGroup);
}
Example 9
| Source File: CheckpointStatsTracker.java From flink with Apache License 2.0 | 5 votes |
|
/**
* Creates an empty map with a {@link TaskStateStats} instance per task
* that is involved in the checkpoint.
*
* @return An empty map with an {@link TaskStateStats} entry for each task that is involved in the checkpoint.
*/
private ConcurrentHashMap<JobVertexID, TaskStateStats> createEmptyTaskStateStatsMap() {
ConcurrentHashMap<JobVertexID, TaskStateStats> taskStatsMap = new ConcurrentHashMap<>(jobVertices.size());
for (ExecutionJobVertex vertex : jobVertices) {
TaskStateStats taskStats = new TaskStateStats(vertex.getJobVertexId(), vertex.getParallelism());
taskStatsMap.put(vertex.getJobVertexId(), taskStats);
}
return taskStatsMap;
}
Example 10
| Source File: CheckpointStatsTracker.java From flink with Apache License 2.0 | 5 votes |
|
/**
* Creates a new checkpoint stats tracker.
*
* @param numRememberedCheckpoints Maximum number of checkpoints to remember, including in progress ones.
* @param jobVertices Job vertices involved in the checkpoints.
* @param jobCheckpointingConfiguration Checkpointing configuration.
* @param metricGroup Metric group for exposed metrics
*/
public CheckpointStatsTracker(
int numRememberedCheckpoints,
List<ExecutionJobVertex> jobVertices,
CheckpointCoordinatorConfiguration jobCheckpointingConfiguration,
MetricGroup metricGroup) {
checkArgument(numRememberedCheckpoints >= 0, "Negative number of remembered checkpoints");
this.history = new CheckpointStatsHistory(numRememberedCheckpoints);
this.jobVertices = checkNotNull(jobVertices, "JobVertices");
this.jobCheckpointingConfiguration = checkNotNull(jobCheckpointingConfiguration);
// Compute the total subtask count. We do this here in order to only
// do it once.
int count = 0;
for (ExecutionJobVertex vertex : jobVertices) {
count += vertex.getParallelism();
}
this.totalSubtaskCount = count;
// Latest snapshot is empty
latestSnapshot = new CheckpointStatsSnapshot(
counts.createSnapshot(),
summary.createSnapshot(),
history.createSnapshot(),
null);
// Register the metrics
registerMetrics(metricGroup);
}
Example 11
| Source File: StateAssignmentOperation.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
/**
* Verifies conditions in regards to parallelism and maxParallelism that must be met when restoring state.
*
* @param operatorState state to restore
* @param executionJobVertex task for which the state should be restored
*/
private static void checkParallelismPreconditions(OperatorState operatorState, ExecutionJobVertex executionJobVertex) {
//----------------------------------------max parallelism preconditions-------------------------------------
if (operatorState.getMaxParallelism() < executionJobVertex.getParallelism()) {
throw new IllegalStateException("The state for task " + executionJobVertex.getJobVertexId() +
" can not be restored. The maximum parallelism (" + operatorState.getMaxParallelism() +
") of the restored state is lower than the configured parallelism (" + executionJobVertex.getParallelism() +
"). Please reduce the parallelism of the task to be lower or equal to the maximum parallelism."
);
}
// check that the number of key groups have not changed or if we need to override it to satisfy the restored state
if (operatorState.getMaxParallelism() != executionJobVertex.getMaxParallelism()) {
if (!executionJobVertex.isMaxParallelismConfigured()) {
// if the max parallelism was not explicitly specified by the user, we derive it from the state
LOG.debug("Overriding maximum parallelism for JobVertex {} from {} to {}",
executionJobVertex.getJobVertexId(), executionJobVertex.getMaxParallelism(), operatorState.getMaxParallelism());
executionJobVertex.setMaxParallelism(operatorState.getMaxParallelism());
} else {
// if the max parallelism was explicitly specified, we complain on mismatch
throw new IllegalStateException("The maximum parallelism (" +
operatorState.getMaxParallelism() + ") with which the latest " +
"checkpoint of the execution job vertex " + executionJobVertex +
" has been taken and the current maximum parallelism (" +
executionJobVertex.getMaxParallelism() + ") changed. This " +
"is currently not supported.");
}
}
}
Example 12
| Source File: CheckpointStatsTracker.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
/**
* Creates an empty map with a {@link TaskStateStats} instance per task
* that is involved in the checkpoint.
*
* @return An empty map with an {@link TaskStateStats} entry for each task that is involved in the checkpoint.
*/
private ConcurrentHashMap<JobVertexID, TaskStateStats> createEmptyTaskStateStatsMap() {
ConcurrentHashMap<JobVertexID, TaskStateStats> taskStatsMap = new ConcurrentHashMap<>(jobVertices.size());
for (ExecutionJobVertex vertex : jobVertices) {
TaskStateStats taskStats = new TaskStateStats(vertex.getJobVertexId(), vertex.getParallelism());
taskStatsMap.put(vertex.getJobVertexId(), taskStats);
}
return taskStatsMap;
}
Example 13
| Source File: StateAssignmentOperation.java From flink with Apache License 2.0 | 4 votes |
|
private void assignAttemptState(ExecutionJobVertex executionJobVertex, List<OperatorState> operatorStates) {
List<OperatorID> operatorIDs = executionJobVertex.getOperatorIDs();
//1. first compute the new parallelism
checkParallelismPreconditions(operatorStates, executionJobVertex);
int newParallelism = executionJobVertex.getParallelism();
List<KeyGroupRange> keyGroupPartitions = createKeyGroupPartitions(
executionJobVertex.getMaxParallelism(),
newParallelism);
final int expectedNumberOfSubTasks = newParallelism * operatorIDs.size();
/*
* Redistribute ManagedOperatorStates and RawOperatorStates from old parallelism to new parallelism.
*
* The old ManagedOperatorStates with old parallelism 3:
*
* parallelism0 parallelism1 parallelism2
* op0 states0,0 state0,1 state0,2
* op1
* op2 states2,0 state2,1 state1,2
* op3 states3,0 state3,1 state3,2
*
* The new ManagedOperatorStates with new parallelism 4:
*
* parallelism0 parallelism1 parallelism2 parallelism3
* op0 state0,0 state0,1 state0,2 state0,3
* op1
* op2 state2,0 state2,1 state2,2 state2,3
* op3 state3,0 state3,1 state3,2 state3,3
*/
Map<OperatorInstanceID, List<OperatorStateHandle>> newManagedOperatorStates =
new HashMap<>(expectedNumberOfSubTasks);
Map<OperatorInstanceID, List<OperatorStateHandle>> newRawOperatorStates =
new HashMap<>(expectedNumberOfSubTasks);
reDistributePartitionableStates(
operatorStates,
newParallelism,
operatorIDs,
newManagedOperatorStates,
newRawOperatorStates);
Map<OperatorInstanceID, List<KeyedStateHandle>> newManagedKeyedState =
new HashMap<>(expectedNumberOfSubTasks);
Map<OperatorInstanceID, List<KeyedStateHandle>> newRawKeyedState =
new HashMap<>(expectedNumberOfSubTasks);
reDistributeKeyedStates(
operatorStates,
newParallelism,
operatorIDs,
keyGroupPartitions,
newManagedKeyedState,
newRawKeyedState);
/*
* An executionJobVertex's all state handles needed to restore are something like a matrix
*
* parallelism0 parallelism1 parallelism2 parallelism3
* op0 sh(0,0) sh(0,1) sh(0,2) sh(0,3)
* op1 sh(1,0) sh(1,1) sh(1,2) sh(1,3)
* op2 sh(2,0) sh(2,1) sh(2,2) sh(2,3)
* op3 sh(3,0) sh(3,1) sh(3,2) sh(3,3)
*
*/
assignTaskStateToExecutionJobVertices(
executionJobVertex,
newManagedOperatorStates,
newRawOperatorStates,
newManagedKeyedState,
newRawKeyedState,
newParallelism);
}
Example 14
| Source File: CheckpointCoordinatorTest.java From flink with Apache License 2.0 | 4 votes |
|
private void performIncrementalCheckpoint(
JobID jid,
CheckpointCoordinator coord,
ExecutionJobVertex jobVertex1,
List<KeyGroupRange> keyGroupPartitions1,
long timestamp,
int cpSequenceNumber) throws Exception {
// trigger the checkpoint
coord.triggerCheckpoint(timestamp, false);
assertTrue(coord.getPendingCheckpoints().keySet().size() == 1);
long checkpointId = Iterables.getOnlyElement(coord.getPendingCheckpoints().keySet());
for (int index = 0; index < jobVertex1.getParallelism(); index++) {
KeyGroupRange keyGroupRange = keyGroupPartitions1.get(index);
Map<StateHandleID, StreamStateHandle> privateState = new HashMap<>();
privateState.put(
new StateHandleID("private-1"),
spy(new ByteStreamStateHandle("private-1", new byte[]{'p'})));
Map<StateHandleID, StreamStateHandle> sharedState = new HashMap<>();
// let all but the first CP overlap by one shared state.
if (cpSequenceNumber > 0) {
sharedState.put(
new StateHandleID("shared-" + (cpSequenceNumber - 1)),
spy(new PlaceholderStreamStateHandle()));
}
sharedState.put(
new StateHandleID("shared-" + cpSequenceNumber),
spy(new ByteStreamStateHandle("shared-" + cpSequenceNumber + "-" + keyGroupRange, new byte[]{'s'})));
IncrementalRemoteKeyedStateHandle managedState =
spy(new IncrementalRemoteKeyedStateHandle(
new UUID(42L, 42L),
keyGroupRange,
checkpointId,
sharedState,
privateState,
spy(new ByteStreamStateHandle("meta", new byte[]{'m'}))));
OperatorSubtaskState operatorSubtaskState =
spy(new OperatorSubtaskState(
StateObjectCollection.empty(),
StateObjectCollection.empty(),
StateObjectCollection.singleton(managedState),
StateObjectCollection.empty()));
Map<OperatorID, OperatorSubtaskState> opStates = new HashMap<>();
opStates.put(jobVertex1.getOperatorIDs().get(0), operatorSubtaskState);
TaskStateSnapshot taskStateSnapshot = new TaskStateSnapshot(opStates);
AcknowledgeCheckpoint acknowledgeCheckpoint = new AcknowledgeCheckpoint(
jid,
jobVertex1.getTaskVertices()[index].getCurrentExecutionAttempt().getAttemptId(),
checkpointId,
new CheckpointMetrics(),
taskStateSnapshot);
coord.receiveAcknowledgeMessage(acknowledgeCheckpoint, TASK_MANAGER_LOCATION_INFO);
}
}
Example 15
| Source File: CheckpointCoordinatorTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
private void performIncrementalCheckpoint(
JobID jid,
CheckpointCoordinator coord,
ExecutionJobVertex jobVertex1,
List<KeyGroupRange> keyGroupPartitions1,
long timestamp,
int cpSequenceNumber) throws Exception {
// trigger the checkpoint
coord.triggerCheckpoint(timestamp, false);
assertTrue(coord.getPendingCheckpoints().keySet().size() == 1);
long checkpointId = Iterables.getOnlyElement(coord.getPendingCheckpoints().keySet());
for (int index = 0; index < jobVertex1.getParallelism(); index++) {
KeyGroupRange keyGroupRange = keyGroupPartitions1.get(index);
Map<StateHandleID, StreamStateHandle> privateState = new HashMap<>();
privateState.put(
new StateHandleID("private-1"),
spy(new ByteStreamStateHandle("private-1", new byte[]{'p'})));
Map<StateHandleID, StreamStateHandle> sharedState = new HashMap<>();
// let all but the first CP overlap by one shared state.
if (cpSequenceNumber > 0) {
sharedState.put(
new StateHandleID("shared-" + (cpSequenceNumber - 1)),
spy(new PlaceholderStreamStateHandle()));
}
sharedState.put(
new StateHandleID("shared-" + cpSequenceNumber),
spy(new ByteStreamStateHandle("shared-" + cpSequenceNumber + "-" + keyGroupRange, new byte[]{'s'})));
IncrementalRemoteKeyedStateHandle managedState =
spy(new IncrementalRemoteKeyedStateHandle(
new UUID(42L, 42L),
keyGroupRange,
checkpointId,
sharedState,
privateState,
spy(new ByteStreamStateHandle("meta", new byte[]{'m'}))));
OperatorSubtaskState operatorSubtaskState =
spy(new OperatorSubtaskState(
StateObjectCollection.empty(),
StateObjectCollection.empty(),
StateObjectCollection.singleton(managedState),
StateObjectCollection.empty()));
Map<OperatorID, OperatorSubtaskState> opStates = new HashMap<>();
opStates.put(jobVertex1.getOperatorIDs().get(0), operatorSubtaskState);
TaskStateSnapshot taskStateSnapshot = new TaskStateSnapshot(opStates);
AcknowledgeCheckpoint acknowledgeCheckpoint = new AcknowledgeCheckpoint(
jid,
jobVertex1.getTaskVertices()[index].getCurrentExecutionAttempt().getAttemptId(),
checkpointId,
new CheckpointMetrics(),
taskStateSnapshot);
coord.receiveAcknowledgeMessage(acknowledgeCheckpoint);
}
}
Example 16
| Source File: StateAssignmentOperation.java From flink with Apache License 2.0 | 4 votes |
|
private void assignAttemptState(ExecutionJobVertex executionJobVertex, List<OperatorState> operatorStates) {
List<OperatorIDPair> operatorIDs = executionJobVertex.getOperatorIDs();
//1. first compute the new parallelism
checkParallelismPreconditions(operatorStates, executionJobVertex);
int newParallelism = executionJobVertex.getParallelism();
List<KeyGroupRange> keyGroupPartitions = createKeyGroupPartitions(
executionJobVertex.getMaxParallelism(),
newParallelism);
final int expectedNumberOfSubTasks = newParallelism * operatorIDs.size();
/*
* Redistribute ManagedOperatorStates and RawOperatorStates from old parallelism to new parallelism.
*
* The old ManagedOperatorStates with old parallelism 3:
*
* parallelism0 parallelism1 parallelism2
* op0 states0,0 state0,1 state0,2
* op1
* op2 states2,0 state2,1 state1,2
* op3 states3,0 state3,1 state3,2
*
* The new ManagedOperatorStates with new parallelism 4:
*
* parallelism0 parallelism1 parallelism2 parallelism3
* op0 state0,0 state0,1 state0,2 state0,3
* op1
* op2 state2,0 state2,1 state2,2 state2,3
* op3 state3,0 state3,1 state3,2 state3,3
*/
Map<OperatorInstanceID, List<OperatorStateHandle>> newManagedOperatorStates = reDistributePartitionableStates(
operatorStates,
newParallelism,
operatorIDs,
OperatorSubtaskState::getManagedOperatorState,
RoundRobinOperatorStateRepartitioner.INSTANCE);
Map<OperatorInstanceID, List<OperatorStateHandle>> newRawOperatorStates = reDistributePartitionableStates(
operatorStates,
newParallelism,
operatorIDs,
OperatorSubtaskState::getRawOperatorState,
RoundRobinOperatorStateRepartitioner.INSTANCE);
final Map<OperatorInstanceID, List<InputChannelStateHandle>> newInputChannelState = reDistributePartitionableStates(
operatorStates,
newParallelism,
operatorIDs,
OperatorSubtaskState::getInputChannelState,
channelStateNonRescalingRepartitioner("input channel"));
final Map<OperatorInstanceID, List<ResultSubpartitionStateHandle>> newResultSubpartitionState = reDistributePartitionableStates(
operatorStates,
newParallelism,
operatorIDs,
OperatorSubtaskState::getResultSubpartitionState,
channelStateNonRescalingRepartitioner("result subpartition"));
Map<OperatorInstanceID, List<KeyedStateHandle>> newManagedKeyedState = new HashMap<>(expectedNumberOfSubTasks);
Map<OperatorInstanceID, List<KeyedStateHandle>> newRawKeyedState = new HashMap<>(expectedNumberOfSubTasks);
reDistributeKeyedStates(
operatorStates,
newParallelism,
operatorIDs,
keyGroupPartitions,
newManagedKeyedState,
newRawKeyedState);
/*
* An executionJobVertex's all state handles needed to restore are something like a matrix
*
* parallelism0 parallelism1 parallelism2 parallelism3
* op0 sh(0,0) sh(0,1) sh(0,2) sh(0,3)
* op1 sh(1,0) sh(1,1) sh(1,2) sh(1,3)
* op2 sh(2,0) sh(2,1) sh(2,2) sh(2,3)
* op3 sh(3,0) sh(3,1) sh(3,2) sh(3,3)
*
*/
assignTaskStateToExecutionJobVertices(
executionJobVertex,
newManagedOperatorStates,
newRawOperatorStates,
newInputChannelState,
newResultSubpartitionState,
newManagedKeyedState,
newRawKeyedState,
newParallelism);
}
Example 17
| Source File: StateAssignmentOperation.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
private void assignAttemptState(ExecutionJobVertex executionJobVertex, List<OperatorState> operatorStates) {
List<OperatorID> operatorIDs = executionJobVertex.getOperatorIDs();
//1. first compute the new parallelism
checkParallelismPreconditions(operatorStates, executionJobVertex);
int newParallelism = executionJobVertex.getParallelism();
List<KeyGroupRange> keyGroupPartitions = createKeyGroupPartitions(
executionJobVertex.getMaxParallelism(),
newParallelism);
final int expectedNumberOfSubTasks = newParallelism * operatorIDs.size();
/*
* Redistribute ManagedOperatorStates and RawOperatorStates from old parallelism to new parallelism.
*
* The old ManagedOperatorStates with old parallelism 3:
*
* parallelism0 parallelism1 parallelism2
* op0 states0,0 state0,1 state0,2
* op1
* op2 states2,0 state2,1 state1,2
* op3 states3,0 state3,1 state3,2
*
* The new ManagedOperatorStates with new parallelism 4:
*
* parallelism0 parallelism1 parallelism2 parallelism3
* op0 state0,0 state0,1 state0,2 state0,3
* op1
* op2 state2,0 state2,1 state2,2 state2,3
* op3 state3,0 state3,1 state3,2 state3,3
*/
Map<OperatorInstanceID, List<OperatorStateHandle>> newManagedOperatorStates =
new HashMap<>(expectedNumberOfSubTasks);
Map<OperatorInstanceID, List<OperatorStateHandle>> newRawOperatorStates =
new HashMap<>(expectedNumberOfSubTasks);
reDistributePartitionableStates(
operatorStates,
newParallelism,
operatorIDs,
newManagedOperatorStates,
newRawOperatorStates);
Map<OperatorInstanceID, List<KeyedStateHandle>> newManagedKeyedState =
new HashMap<>(expectedNumberOfSubTasks);
Map<OperatorInstanceID, List<KeyedStateHandle>> newRawKeyedState =
new HashMap<>(expectedNumberOfSubTasks);
reDistributeKeyedStates(
operatorStates,
newParallelism,
operatorIDs,
keyGroupPartitions,
newManagedKeyedState,
newRawKeyedState);
/*
* An executionJobVertex's all state handles needed to restore are something like a matrix
*
* parallelism0 parallelism1 parallelism2 parallelism3
* op0 sh(0,0) sh(0,1) sh(0,2) sh(0,3)
* op1 sh(1,0) sh(1,1) sh(1,2) sh(1,3)
* op2 sh(2,0) sh(2,1) sh(2,2) sh(2,3)
* op3 sh(3,0) sh(3,1) sh(3,2) sh(3,3)
*
*/
assignTaskStateToExecutionJobVertices(
executionJobVertex,
newManagedOperatorStates,
newRawOperatorStates,
newManagedKeyedState,
newRawKeyedState,
newParallelism);
}
Example 18
| Source File: CheckpointCoordinatorTest.java From flink with Apache License 2.0 | 4 votes |
|
private void performIncrementalCheckpoint(
JobID jid,
CheckpointCoordinator coord,
ExecutionJobVertex jobVertex1,
List<KeyGroupRange> keyGroupPartitions1,
int cpSequenceNumber) throws Exception {
// trigger the checkpoint
coord.triggerCheckpoint(false);
manuallyTriggeredScheduledExecutor.triggerAll();
assertEquals(1, coord.getPendingCheckpoints().size());
long checkpointId = Iterables.getOnlyElement(coord.getPendingCheckpoints().keySet());
for (int index = 0; index < jobVertex1.getParallelism(); index++) {
KeyGroupRange keyGroupRange = keyGroupPartitions1.get(index);
Map<StateHandleID, StreamStateHandle> privateState = new HashMap<>();
privateState.put(
new StateHandleID("private-1"),
spy(new ByteStreamStateHandle("private-1", new byte[]{'p'})));
Map<StateHandleID, StreamStateHandle> sharedState = new HashMap<>();
// let all but the first CP overlap by one shared state.
if (cpSequenceNumber > 0) {
sharedState.put(
new StateHandleID("shared-" + (cpSequenceNumber - 1)),
spy(new PlaceholderStreamStateHandle()));
}
sharedState.put(
new StateHandleID("shared-" + cpSequenceNumber),
spy(new ByteStreamStateHandle("shared-" + cpSequenceNumber + "-" + keyGroupRange, new byte[]{'s'})));
IncrementalRemoteKeyedStateHandle managedState =
spy(new IncrementalRemoteKeyedStateHandle(
new UUID(42L, 42L),
keyGroupRange,
checkpointId,
sharedState,
privateState,
spy(new ByteStreamStateHandle("meta", new byte[]{'m'}))));
OperatorSubtaskState operatorSubtaskState =
spy(new OperatorSubtaskState(
StateObjectCollection.empty(),
StateObjectCollection.empty(),
StateObjectCollection.singleton(managedState),
StateObjectCollection.empty()));
Map<OperatorID, OperatorSubtaskState> opStates = new HashMap<>();
opStates.put(jobVertex1.getOperatorIDs().get(0).getGeneratedOperatorID(), operatorSubtaskState);
TaskStateSnapshot taskStateSnapshot = new TaskStateSnapshot(opStates);
AcknowledgeCheckpoint acknowledgeCheckpoint = new AcknowledgeCheckpoint(
jid,
jobVertex1.getTaskVertices()[index].getCurrentExecutionAttempt().getAttemptId(),
checkpointId,
new CheckpointMetrics(),
taskStateSnapshot);
coord.receiveAcknowledgeMessage(acknowledgeCheckpoint, TASK_MANAGER_LOCATION_INFO);
}
}
