org.apache.flink.runtime.jobgraph.OperatorInstanceID Java Examples
The following examples show how to use
org.apache.flink.runtime.jobgraph.OperatorInstanceID.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
| Source File: StateAssignmentOperation.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
|
public static OperatorSubtaskState operatorSubtaskStateFrom(
OperatorInstanceID instanceID,
Map<OperatorInstanceID, List<OperatorStateHandle>> subManagedOperatorState,
Map<OperatorInstanceID, List<OperatorStateHandle>> subRawOperatorState,
Map<OperatorInstanceID, List<KeyedStateHandle>> subManagedKeyedState,
Map<OperatorInstanceID, List<KeyedStateHandle>> subRawKeyedState) {
if (!subManagedOperatorState.containsKey(instanceID) &&
!subRawOperatorState.containsKey(instanceID) &&
!subManagedKeyedState.containsKey(instanceID) &&
!subRawKeyedState.containsKey(instanceID)) {
return new OperatorSubtaskState();
}
if (!subManagedKeyedState.containsKey(instanceID)) {
checkState(!subRawKeyedState.containsKey(instanceID));
}
return new OperatorSubtaskState(
new StateObjectCollection<>(subManagedOperatorState.getOrDefault(instanceID, Collections.emptyList())),
new StateObjectCollection<>(subRawOperatorState.getOrDefault(instanceID, Collections.emptyList())),
new StateObjectCollection<>(subManagedKeyedState.getOrDefault(instanceID, Collections.emptyList())),
new StateObjectCollection<>(subRawKeyedState.getOrDefault(instanceID, Collections.emptyList())));
}
Example #2
| Source File: StateAssignmentOperation.java From flink with Apache License 2.0 | 6 votes |
|
public static <T extends StateObject> Map<OperatorInstanceID, List<T>> applyRepartitioner(
OperatorID operatorID,
OperatorStateRepartitioner<T> opStateRepartitioner,
List<List<T>> chainOpParallelStates,
int oldParallelism,
int newParallelism) {
List<List<T>> states = applyRepartitioner(
opStateRepartitioner,
chainOpParallelStates,
oldParallelism,
newParallelism);
Map<OperatorInstanceID, List<T>> result = new HashMap<>(states.size());
for (int subtaskIndex = 0; subtaskIndex < states.size(); subtaskIndex++) {
checkNotNull(states.get(subtaskIndex) != null, "states.get(subtaskIndex) is null");
result.put(OperatorInstanceID.of(subtaskIndex, operatorID), states.get(subtaskIndex));
}
return result;
}
Example #3
| Source File: StateAssignmentOperation.java From flink with Apache License 2.0 | 6 votes |
|
public static <T extends StateObject> Map<OperatorInstanceID, List<T>> reDistributePartitionableStates(
List<OperatorState> oldOperatorStates,
int newParallelism,
List<OperatorIDPair> newOperatorIDs,
Function<OperatorSubtaskState, StateObjectCollection<T>> extractHandle,
OperatorStateRepartitioner<T> stateRepartitioner) {
//TODO: rewrite this method to only use OperatorID
checkState(newOperatorIDs.size() == oldOperatorStates.size(),
"This method still depends on the order of the new and old operators");
// The nested list wraps as the level of operator -> subtask -> state object collection
List<List<List<T>>> oldStates = splitManagedAndRawOperatorStates(oldOperatorStates, extractHandle);
Map<OperatorInstanceID, List<T>> result = new HashMap<>();
for (int operatorIndex = 0; operatorIndex < newOperatorIDs.size(); operatorIndex++) {
result.putAll(applyRepartitioner(
newOperatorIDs.get(operatorIndex).getGeneratedOperatorID(),
stateRepartitioner,
oldStates.get(operatorIndex),
oldOperatorStates.get(operatorIndex).getParallelism(),
newParallelism));
}
return result;
}
Example #4
| Source File: StateAssignmentOperation.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
|
public static Map<OperatorInstanceID, List<OperatorStateHandle>> applyRepartitioner(
OperatorID operatorID,
OperatorStateRepartitioner opStateRepartitioner,
List<List<OperatorStateHandle>> chainOpParallelStates,
int oldParallelism,
int newParallelism) {
List<List<OperatorStateHandle>> states = applyRepartitioner(
opStateRepartitioner,
chainOpParallelStates,
oldParallelism,
newParallelism);
Map<OperatorInstanceID, List<OperatorStateHandle>> result = new HashMap<>(states.size());
for (int subtaskIndex = 0; subtaskIndex < states.size(); subtaskIndex++) {
checkNotNull(states.get(subtaskIndex) != null, "states.get(subtaskIndex) is null");
result.put(OperatorInstanceID.of(subtaskIndex, operatorID), states.get(subtaskIndex));
}
return result;
}
Example #5
| Source File: StateAssignmentOperation.java From flink with Apache License 2.0 | 6 votes |
|
public static OperatorSubtaskState operatorSubtaskStateFrom(
OperatorInstanceID instanceID,
Map<OperatorInstanceID, List<OperatorStateHandle>> subManagedOperatorState,
Map<OperatorInstanceID, List<OperatorStateHandle>> subRawOperatorState,
Map<OperatorInstanceID, List<KeyedStateHandle>> subManagedKeyedState,
Map<OperatorInstanceID, List<KeyedStateHandle>> subRawKeyedState) {
if (!subManagedOperatorState.containsKey(instanceID) &&
!subRawOperatorState.containsKey(instanceID) &&
!subManagedKeyedState.containsKey(instanceID) &&
!subRawKeyedState.containsKey(instanceID)) {
return new OperatorSubtaskState();
}
if (!subManagedKeyedState.containsKey(instanceID)) {
checkState(!subRawKeyedState.containsKey(instanceID));
}
return new OperatorSubtaskState(
new StateObjectCollection<>(subManagedOperatorState.getOrDefault(instanceID, Collections.emptyList())),
new StateObjectCollection<>(subRawOperatorState.getOrDefault(instanceID, Collections.emptyList())),
new StateObjectCollection<>(subManagedKeyedState.getOrDefault(instanceID, Collections.emptyList())),
new StateObjectCollection<>(subRawKeyedState.getOrDefault(instanceID, Collections.emptyList())));
}
Example #6
| Source File: StateAssignmentOperation.java From flink with Apache License 2.0 | 6 votes |
|
public static Map<OperatorInstanceID, List<OperatorStateHandle>> applyRepartitioner(
OperatorID operatorID,
OperatorStateRepartitioner opStateRepartitioner,
List<List<OperatorStateHandle>> chainOpParallelStates,
int oldParallelism,
int newParallelism) {
List<List<OperatorStateHandle>> states = applyRepartitioner(
opStateRepartitioner,
chainOpParallelStates,
oldParallelism,
newParallelism);
Map<OperatorInstanceID, List<OperatorStateHandle>> result = new HashMap<>(states.size());
for (int subtaskIndex = 0; subtaskIndex < states.size(); subtaskIndex++) {
checkNotNull(states.get(subtaskIndex) != null, "states.get(subtaskIndex) is null");
result.put(OperatorInstanceID.of(subtaskIndex, operatorID), states.get(subtaskIndex));
}
return result;
}
Example #7
| Source File: StateAssignmentOperation.java From flink with Apache License 2.0 | 5 votes |
|
private void reDistributeKeyedStates(
List<OperatorState> oldOperatorStates,
int newParallelism,
List<OperatorID> newOperatorIDs,
List<KeyGroupRange> newKeyGroupPartitions,
Map<OperatorInstanceID, List<KeyedStateHandle>> newManagedKeyedState,
Map<OperatorInstanceID, List<KeyedStateHandle>> newRawKeyedState) {
//TODO: rewrite this method to only use OperatorID
checkState(newOperatorIDs.size() == oldOperatorStates.size(),
"This method still depends on the order of the new and old operators");
for (int operatorIndex = 0; operatorIndex < newOperatorIDs.size(); operatorIndex++) {
OperatorState operatorState = oldOperatorStates.get(operatorIndex);
int oldParallelism = operatorState.getParallelism();
for (int subTaskIndex = 0; subTaskIndex < newParallelism; subTaskIndex++) {
OperatorInstanceID instanceID = OperatorInstanceID.of(subTaskIndex, newOperatorIDs.get(operatorIndex));
Tuple2<List<KeyedStateHandle>, List<KeyedStateHandle>> subKeyedStates = reAssignSubKeyedStates(
operatorState,
newKeyGroupPartitions,
subTaskIndex,
newParallelism,
oldParallelism);
newManagedKeyedState.put(instanceID, subKeyedStates.f0);
newRawKeyedState.put(instanceID, subKeyedStates.f1);
}
}
}
Example #8
| Source File: RestoreStreamTaskTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testRestoreAfterScaleUp() throws Exception {
OperatorID headOperatorID = new OperatorID(42L, 42L);
OperatorID tailOperatorID = new OperatorID(44L, 44L);
JobManagerTaskRestore restore = createRunAndCheckpointOperatorChain(
headOperatorID,
new CounterOperator(),
tailOperatorID,
new CounterOperator(),
Optional.empty());
TaskStateSnapshot stateHandles = restore.getTaskStateSnapshot();
assertEquals(2, stateHandles.getSubtaskStateMappings().size());
// test empty state in case of scale up
OperatorSubtaskState emptyHeadOperatorState = StateAssignmentOperation.operatorSubtaskStateFrom(
new OperatorInstanceID(0, headOperatorID),
Collections.emptyMap(),
Collections.emptyMap(),
Collections.emptyMap(),
Collections.emptyMap(),
Collections.emptyMap(),
Collections.emptyMap());
stateHandles.putSubtaskStateByOperatorID(headOperatorID, emptyHeadOperatorState);
createRunAndCheckpointOperatorChain(
headOperatorID,
new CounterOperator(),
tailOperatorID,
new CounterOperator(),
Optional.of(restore));
assertEquals(new HashSet<>(Arrays.asList(headOperatorID, tailOperatorID)), RESTORED_OPERATORS);
}
Example #9
| Source File: StateAssignmentOperation.java From flink with Apache License 2.0 | 5 votes |
|
private void reDistributeKeyedStates(
List<OperatorState> oldOperatorStates,
int newParallelism,
List<OperatorIDPair> newOperatorIDs,
List<KeyGroupRange> newKeyGroupPartitions,
Map<OperatorInstanceID, List<KeyedStateHandle>> newManagedKeyedState,
Map<OperatorInstanceID, List<KeyedStateHandle>> newRawKeyedState) {
//TODO: rewrite this method to only use OperatorID
checkState(newOperatorIDs.size() == oldOperatorStates.size(),
"This method still depends on the order of the new and old operators");
for (int operatorIndex = 0; operatorIndex < newOperatorIDs.size(); operatorIndex++) {
OperatorState operatorState = oldOperatorStates.get(operatorIndex);
int oldParallelism = operatorState.getParallelism();
for (int subTaskIndex = 0; subTaskIndex < newParallelism; subTaskIndex++) {
OperatorInstanceID instanceID = OperatorInstanceID.of(subTaskIndex, newOperatorIDs.get(operatorIndex).getGeneratedOperatorID());
Tuple2<List<KeyedStateHandle>, List<KeyedStateHandle>> subKeyedStates = reAssignSubKeyedStates(
operatorState,
newKeyGroupPartitions,
subTaskIndex,
newParallelism,
oldParallelism);
newManagedKeyedState.put(instanceID, subKeyedStates.f0);
newRawKeyedState.put(instanceID, subKeyedStates.f1);
}
}
}
Example #10
| Source File: StateAssignmentOperation.java From flink with Apache License 2.0 | 5 votes |
|
public static OperatorSubtaskState operatorSubtaskStateFrom(
OperatorInstanceID instanceID,
Map<OperatorInstanceID, List<OperatorStateHandle>> subManagedOperatorState,
Map<OperatorInstanceID, List<OperatorStateHandle>> subRawOperatorState,
Map<OperatorInstanceID, List<InputChannelStateHandle>> inputChannelStates,
Map<OperatorInstanceID, List<ResultSubpartitionStateHandle>> resultSubpartitionStates,
Map<OperatorInstanceID, List<KeyedStateHandle>> subManagedKeyedState,
Map<OperatorInstanceID, List<KeyedStateHandle>> subRawKeyedState) {
if (!subManagedOperatorState.containsKey(instanceID) &&
!subRawOperatorState.containsKey(instanceID) &&
!inputChannelStates.containsKey(instanceID) &&
!resultSubpartitionStates.containsKey(instanceID) &&
!subManagedKeyedState.containsKey(instanceID) &&
!subRawKeyedState.containsKey(instanceID)) {
return new OperatorSubtaskState();
}
if (!subManagedKeyedState.containsKey(instanceID)) {
checkState(!subRawKeyedState.containsKey(instanceID));
}
return new OperatorSubtaskState(
new StateObjectCollection<>(subManagedOperatorState.getOrDefault(instanceID, emptyList())),
new StateObjectCollection<>(subRawOperatorState.getOrDefault(instanceID, emptyList())),
new StateObjectCollection<>(subManagedKeyedState.getOrDefault(instanceID, emptyList())),
new StateObjectCollection<>(subRawKeyedState.getOrDefault(instanceID, emptyList())),
new StateObjectCollection<>(inputChannelStates.getOrDefault(instanceID, emptyList())),
new StateObjectCollection<>(resultSubpartitionStates.getOrDefault(instanceID, emptyList())));
}
Example #11
| Source File: OperatorStateInputFormat.java From flink with Apache License 2.0 | 5 votes |
|
private OperatorStateInputSplit[] getOperatorStateInputSplits(int minNumSplits) {
Map<OperatorInstanceID, List<OperatorStateHandle>> newManagedOperatorStates = reDistributePartitionableStates(
singletonList(operatorState),
minNumSplits,
singletonList(OperatorIDPair.generatedIDOnly(operatorState.getOperatorID())),
OperatorSubtaskState::getManagedOperatorState,
RoundRobinOperatorStateRepartitioner.INSTANCE);
return CollectionUtil.mapWithIndex(
newManagedOperatorStates.values(),
(handles, index) -> new OperatorStateInputSplit(new StateObjectCollection<>(handles), index)
).toArray(OperatorStateInputSplit[]::new);
}
Example #12
| Source File: RestoreStreamTaskTest.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testRestoreAfterScaleUp() throws Exception {
OperatorID headOperatorID = new OperatorID(42L, 42L);
OperatorID tailOperatorID = new OperatorID(44L, 44L);
JobManagerTaskRestore restore = createRunAndCheckpointOperatorChain(
headOperatorID,
new CounterOperator(),
tailOperatorID,
new CounterOperator(),
Optional.empty());
TaskStateSnapshot stateHandles = restore.getTaskStateSnapshot();
assertEquals(2, stateHandles.getSubtaskStateMappings().size());
// test empty state in case of scale up
OperatorSubtaskState emptyHeadOperatorState = StateAssignmentOperation.operatorSubtaskStateFrom(
new OperatorInstanceID(0, headOperatorID),
Collections.emptyMap(),
Collections.emptyMap(),
Collections.emptyMap(),
Collections.emptyMap());
stateHandles.putSubtaskStateByOperatorID(headOperatorID, emptyHeadOperatorState);
createRunAndCheckpointOperatorChain(
headOperatorID,
new CounterOperator(),
tailOperatorID,
new CounterOperator(),
Optional.of(restore));
assertEquals(new HashSet<>(Arrays.asList(headOperatorID, tailOperatorID)), RESTORED_OPERATORS);
}
Example #13
| Source File: StateAssignmentOperation.java From flink with Apache License 2.0 | 5 votes |
|
public static void reDistributePartitionableStates(
List<OperatorState> oldOperatorStates,
int newParallelism,
List<OperatorID> newOperatorIDs,
Map<OperatorInstanceID, List<OperatorStateHandle>> newManagedOperatorStates,
Map<OperatorInstanceID, List<OperatorStateHandle>> newRawOperatorStates) {
//TODO: rewrite this method to only use OperatorID
checkState(newOperatorIDs.size() == oldOperatorStates.size(),
"This method still depends on the order of the new and old operators");
// The nested list wraps as the level of operator -> subtask -> state object collection
List<List<List<OperatorStateHandle>>> oldManagedOperatorStates = new ArrayList<>(oldOperatorStates.size());
List<List<List<OperatorStateHandle>>> oldRawOperatorStates = new ArrayList<>(oldOperatorStates.size());
splitManagedAndRawOperatorStates(oldOperatorStates, oldManagedOperatorStates, oldRawOperatorStates);
OperatorStateRepartitioner opStateRepartitioner = RoundRobinOperatorStateRepartitioner.INSTANCE;
for (int operatorIndex = 0; operatorIndex < newOperatorIDs.size(); operatorIndex++) {
OperatorState operatorState = oldOperatorStates.get(operatorIndex);
int oldParallelism = operatorState.getParallelism();
OperatorID operatorID = newOperatorIDs.get(operatorIndex);
newManagedOperatorStates.putAll(applyRepartitioner(
operatorID,
opStateRepartitioner,
oldManagedOperatorStates.get(operatorIndex),
oldParallelism,
newParallelism));
newRawOperatorStates.putAll(applyRepartitioner(
operatorID,
opStateRepartitioner,
oldRawOperatorStates.get(operatorIndex),
oldParallelism,
newParallelism));
}
}
Example #14
| Source File: StateAssignmentOperation.java From flink with Apache License 2.0 | 5 votes |
|
private void assignTaskStateToExecutionJobVertices(
ExecutionJobVertex executionJobVertex,
Map<OperatorInstanceID, List<OperatorStateHandle>> subManagedOperatorState,
Map<OperatorInstanceID, List<OperatorStateHandle>> subRawOperatorState,
Map<OperatorInstanceID, List<KeyedStateHandle>> subManagedKeyedState,
Map<OperatorInstanceID, List<KeyedStateHandle>> subRawKeyedState,
int newParallelism) {
List<OperatorID> operatorIDs = executionJobVertex.getOperatorIDs();
for (int subTaskIndex = 0; subTaskIndex < newParallelism; subTaskIndex++) {
Execution currentExecutionAttempt = executionJobVertex.getTaskVertices()[subTaskIndex]
.getCurrentExecutionAttempt();
TaskStateSnapshot taskState = new TaskStateSnapshot(operatorIDs.size());
boolean statelessTask = true;
for (OperatorID operatorID : operatorIDs) {
OperatorInstanceID instanceID = OperatorInstanceID.of(subTaskIndex, operatorID);
OperatorSubtaskState operatorSubtaskState = operatorSubtaskStateFrom(
instanceID,
subManagedOperatorState,
subRawOperatorState,
subManagedKeyedState,
subRawKeyedState);
if (operatorSubtaskState.hasState()) {
statelessTask = false;
}
taskState.putSubtaskStateByOperatorID(operatorID, operatorSubtaskState);
}
if (!statelessTask) {
JobManagerTaskRestore taskRestore = new JobManagerTaskRestore(restoreCheckpointId, taskState);
currentExecutionAttempt.setInitialState(taskRestore);
}
}
}
Example #15
| Source File: OperatorStateInputFormat.java From flink with Apache License 2.0 | 5 votes |
|
private OperatorStateInputSplit[] getOperatorStateInputSplits(int minNumSplits) {
final Map<OperatorInstanceID, List<OperatorStateHandle>> newManagedOperatorStates = new HashMap<>();
StateAssignmentOperation.reDistributePartitionableStates(
Collections.singletonList(operatorState),
minNumSplits,
Collections.singletonList(operatorState.getOperatorID()),
newManagedOperatorStates,
new HashMap<>());
return CollectionUtil.mapWithIndex(
newManagedOperatorStates.values(),
(handles, index) -> new OperatorStateInputSplit(new StateObjectCollection<>(handles), index)
).toArray(OperatorStateInputSplit[]::new);
}
Example #16
| Source File: RestoreStreamTaskTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
@Test
public void testRestoreAfterScaleUp() throws Exception {
OperatorID headOperatorID = new OperatorID(42L, 42L);
OperatorID tailOperatorID = new OperatorID(44L, 44L);
JobManagerTaskRestore restore = createRunAndCheckpointOperatorChain(
headOperatorID,
new CounterOperator(),
tailOperatorID,
new CounterOperator(),
Optional.empty());
TaskStateSnapshot stateHandles = restore.getTaskStateSnapshot();
assertEquals(2, stateHandles.getSubtaskStateMappings().size());
// test empty state in case of scale up
OperatorSubtaskState emptyHeadOperatorState = StateAssignmentOperation.operatorSubtaskStateFrom(
new OperatorInstanceID(0, headOperatorID),
Collections.emptyMap(),
Collections.emptyMap(),
Collections.emptyMap(),
Collections.emptyMap());
stateHandles.putSubtaskStateByOperatorID(headOperatorID, emptyHeadOperatorState);
createRunAndCheckpointOperatorChain(
headOperatorID,
new CounterOperator(),
tailOperatorID,
new CounterOperator(),
Optional.of(restore));
assertEquals(new HashSet<>(Arrays.asList(headOperatorID, tailOperatorID)), RESTORED_OPERATORS);
}
Example #17
| Source File: StateAssignmentOperation.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
@VisibleForTesting
static void reDistributePartitionableStates(
List<OperatorState> oldOperatorStates,
int newParallelism,
List<OperatorID> newOperatorIDs,
Map<OperatorInstanceID, List<OperatorStateHandle>> newManagedOperatorStates,
Map<OperatorInstanceID, List<OperatorStateHandle>> newRawOperatorStates) {
//TODO: rewrite this method to only use OperatorID
checkState(newOperatorIDs.size() == oldOperatorStates.size(),
"This method still depends on the order of the new and old operators");
// The nested list wraps as the level of operator -> subtask -> state object collection
List<List<List<OperatorStateHandle>>> oldManagedOperatorStates = new ArrayList<>(oldOperatorStates.size());
List<List<List<OperatorStateHandle>>> oldRawOperatorStates = new ArrayList<>(oldOperatorStates.size());
splitManagedAndRawOperatorStates(oldOperatorStates, oldManagedOperatorStates, oldRawOperatorStates);
OperatorStateRepartitioner opStateRepartitioner = RoundRobinOperatorStateRepartitioner.INSTANCE;
for (int operatorIndex = 0; operatorIndex < newOperatorIDs.size(); operatorIndex++) {
OperatorState operatorState = oldOperatorStates.get(operatorIndex);
int oldParallelism = operatorState.getParallelism();
OperatorID operatorID = newOperatorIDs.get(operatorIndex);
newManagedOperatorStates.putAll(applyRepartitioner(
operatorID,
opStateRepartitioner,
oldManagedOperatorStates.get(operatorIndex),
oldParallelism,
newParallelism));
newRawOperatorStates.putAll(applyRepartitioner(
operatorID,
opStateRepartitioner,
oldRawOperatorStates.get(operatorIndex),
oldParallelism,
newParallelism));
}
}
Example #18
| Source File: StateAssignmentOperation.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
private void reDistributeKeyedStates(
List<OperatorState> oldOperatorStates,
int newParallelism,
List<OperatorID> newOperatorIDs,
List<KeyGroupRange> newKeyGroupPartitions,
Map<OperatorInstanceID, List<KeyedStateHandle>> newManagedKeyedState,
Map<OperatorInstanceID, List<KeyedStateHandle>> newRawKeyedState) {
//TODO: rewrite this method to only use OperatorID
checkState(newOperatorIDs.size() == oldOperatorStates.size(),
"This method still depends on the order of the new and old operators");
for (int operatorIndex = 0; operatorIndex < newOperatorIDs.size(); operatorIndex++) {
OperatorState operatorState = oldOperatorStates.get(operatorIndex);
int oldParallelism = operatorState.getParallelism();
for (int subTaskIndex = 0; subTaskIndex < newParallelism; subTaskIndex++) {
OperatorInstanceID instanceID = OperatorInstanceID.of(subTaskIndex, newOperatorIDs.get(operatorIndex));
Tuple2<List<KeyedStateHandle>, List<KeyedStateHandle>> subKeyedStates = reAssignSubKeyedStates(
operatorState,
newKeyGroupPartitions,
subTaskIndex,
newParallelism,
oldParallelism);
newManagedKeyedState.put(instanceID, subKeyedStates.f0);
newRawKeyedState.put(instanceID, subKeyedStates.f1);
}
}
}
Example #19
| Source File: StateAssignmentOperation.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
private void assignTaskStateToExecutionJobVertices(
ExecutionJobVertex executionJobVertex,
Map<OperatorInstanceID, List<OperatorStateHandle>> subManagedOperatorState,
Map<OperatorInstanceID, List<OperatorStateHandle>> subRawOperatorState,
Map<OperatorInstanceID, List<KeyedStateHandle>> subManagedKeyedState,
Map<OperatorInstanceID, List<KeyedStateHandle>> subRawKeyedState,
int newParallelism) {
List<OperatorID> operatorIDs = executionJobVertex.getOperatorIDs();
for (int subTaskIndex = 0; subTaskIndex < newParallelism; subTaskIndex++) {
Execution currentExecutionAttempt = executionJobVertex.getTaskVertices()[subTaskIndex]
.getCurrentExecutionAttempt();
TaskStateSnapshot taskState = new TaskStateSnapshot(operatorIDs.size());
boolean statelessTask = true;
for (OperatorID operatorID : operatorIDs) {
OperatorInstanceID instanceID = OperatorInstanceID.of(subTaskIndex, operatorID);
OperatorSubtaskState operatorSubtaskState = operatorSubtaskStateFrom(
instanceID,
subManagedOperatorState,
subRawOperatorState,
subManagedKeyedState,
subRawKeyedState);
if (operatorSubtaskState.hasState()) {
statelessTask = false;
}
taskState.putSubtaskStateByOperatorID(operatorID, operatorSubtaskState);
}
if (!statelessTask) {
JobManagerTaskRestore taskRestore = new JobManagerTaskRestore(restoreCheckpointId, taskState);
currentExecutionAttempt.setInitialState(taskRestore);
}
}
}
Example #20
| Source File: StateAssignmentOperation.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
private void assignAttemptState(ExecutionJobVertex executionJobVertex, List<OperatorState> operatorStates) {
List<OperatorID> operatorIDs = executionJobVertex.getOperatorIDs();
//1. first compute the new parallelism
checkParallelismPreconditions(operatorStates, executionJobVertex);
int newParallelism = executionJobVertex.getParallelism();
List<KeyGroupRange> keyGroupPartitions = createKeyGroupPartitions(
executionJobVertex.getMaxParallelism(),
newParallelism);
final int expectedNumberOfSubTasks = newParallelism * operatorIDs.size();
/*
* Redistribute ManagedOperatorStates and RawOperatorStates from old parallelism to new parallelism.
*
* The old ManagedOperatorStates with old parallelism 3:
*
* parallelism0 parallelism1 parallelism2
* op0 states0,0 state0,1 state0,2
* op1
* op2 states2,0 state2,1 state1,2
* op3 states3,0 state3,1 state3,2
*
* The new ManagedOperatorStates with new parallelism 4:
*
* parallelism0 parallelism1 parallelism2 parallelism3
* op0 state0,0 state0,1 state0,2 state0,3
* op1
* op2 state2,0 state2,1 state2,2 state2,3
* op3 state3,0 state3,1 state3,2 state3,3
*/
Map<OperatorInstanceID, List<OperatorStateHandle>> newManagedOperatorStates =
new HashMap<>(expectedNumberOfSubTasks);
Map<OperatorInstanceID, List<OperatorStateHandle>> newRawOperatorStates =
new HashMap<>(expectedNumberOfSubTasks);
reDistributePartitionableStates(
operatorStates,
newParallelism,
operatorIDs,
newManagedOperatorStates,
newRawOperatorStates);
Map<OperatorInstanceID, List<KeyedStateHandle>> newManagedKeyedState =
new HashMap<>(expectedNumberOfSubTasks);
Map<OperatorInstanceID, List<KeyedStateHandle>> newRawKeyedState =
new HashMap<>(expectedNumberOfSubTasks);
reDistributeKeyedStates(
operatorStates,
newParallelism,
operatorIDs,
keyGroupPartitions,
newManagedKeyedState,
newRawKeyedState);
/*
* An executionJobVertex's all state handles needed to restore are something like a matrix
*
* parallelism0 parallelism1 parallelism2 parallelism3
* op0 sh(0,0) sh(0,1) sh(0,2) sh(0,3)
* op1 sh(1,0) sh(1,1) sh(1,2) sh(1,3)
* op2 sh(2,0) sh(2,1) sh(2,2) sh(2,3)
* op3 sh(3,0) sh(3,1) sh(3,2) sh(3,3)
*
*/
assignTaskStateToExecutionJobVertices(
executionJobVertex,
newManagedOperatorStates,
newRawOperatorStates,
newManagedKeyedState,
newRawKeyedState,
newParallelism);
}
Example #21
| Source File: StateAssignmentOperation.java From flink with Apache License 2.0 | 4 votes |
|
private void assignAttemptState(ExecutionJobVertex executionJobVertex, List<OperatorState> operatorStates) {
List<OperatorID> operatorIDs = executionJobVertex.getOperatorIDs();
//1. first compute the new parallelism
checkParallelismPreconditions(operatorStates, executionJobVertex);
int newParallelism = executionJobVertex.getParallelism();
List<KeyGroupRange> keyGroupPartitions = createKeyGroupPartitions(
executionJobVertex.getMaxParallelism(),
newParallelism);
final int expectedNumberOfSubTasks = newParallelism * operatorIDs.size();
/*
* Redistribute ManagedOperatorStates and RawOperatorStates from old parallelism to new parallelism.
*
* The old ManagedOperatorStates with old parallelism 3:
*
* parallelism0 parallelism1 parallelism2
* op0 states0,0 state0,1 state0,2
* op1
* op2 states2,0 state2,1 state1,2
* op3 states3,0 state3,1 state3,2
*
* The new ManagedOperatorStates with new parallelism 4:
*
* parallelism0 parallelism1 parallelism2 parallelism3
* op0 state0,0 state0,1 state0,2 state0,3
* op1
* op2 state2,0 state2,1 state2,2 state2,3
* op3 state3,0 state3,1 state3,2 state3,3
*/
Map<OperatorInstanceID, List<OperatorStateHandle>> newManagedOperatorStates =
new HashMap<>(expectedNumberOfSubTasks);
Map<OperatorInstanceID, List<OperatorStateHandle>> newRawOperatorStates =
new HashMap<>(expectedNumberOfSubTasks);
reDistributePartitionableStates(
operatorStates,
newParallelism,
operatorIDs,
newManagedOperatorStates,
newRawOperatorStates);
Map<OperatorInstanceID, List<KeyedStateHandle>> newManagedKeyedState =
new HashMap<>(expectedNumberOfSubTasks);
Map<OperatorInstanceID, List<KeyedStateHandle>> newRawKeyedState =
new HashMap<>(expectedNumberOfSubTasks);
reDistributeKeyedStates(
operatorStates,
newParallelism,
operatorIDs,
keyGroupPartitions,
newManagedKeyedState,
newRawKeyedState);
/*
* An executionJobVertex's all state handles needed to restore are something like a matrix
*
* parallelism0 parallelism1 parallelism2 parallelism3
* op0 sh(0,0) sh(0,1) sh(0,2) sh(0,3)
* op1 sh(1,0) sh(1,1) sh(1,2) sh(1,3)
* op2 sh(2,0) sh(2,1) sh(2,2) sh(2,3)
* op3 sh(3,0) sh(3,1) sh(3,2) sh(3,3)
*
*/
assignTaskStateToExecutionJobVertices(
executionJobVertex,
newManagedOperatorStates,
newRawOperatorStates,
newManagedKeyedState,
newRawKeyedState,
newParallelism);
}
Example #22
| Source File: StateAssignmentOperation.java From flink with Apache License 2.0 | 4 votes |
|
private void assignAttemptState(ExecutionJobVertex executionJobVertex, List<OperatorState> operatorStates) {
List<OperatorIDPair> operatorIDs = executionJobVertex.getOperatorIDs();
//1. first compute the new parallelism
checkParallelismPreconditions(operatorStates, executionJobVertex);
int newParallelism = executionJobVertex.getParallelism();
List<KeyGroupRange> keyGroupPartitions = createKeyGroupPartitions(
executionJobVertex.getMaxParallelism(),
newParallelism);
final int expectedNumberOfSubTasks = newParallelism * operatorIDs.size();
/*
* Redistribute ManagedOperatorStates and RawOperatorStates from old parallelism to new parallelism.
*
* The old ManagedOperatorStates with old parallelism 3:
*
* parallelism0 parallelism1 parallelism2
* op0 states0,0 state0,1 state0,2
* op1
* op2 states2,0 state2,1 state1,2
* op3 states3,0 state3,1 state3,2
*
* The new ManagedOperatorStates with new parallelism 4:
*
* parallelism0 parallelism1 parallelism2 parallelism3
* op0 state0,0 state0,1 state0,2 state0,3
* op1
* op2 state2,0 state2,1 state2,2 state2,3
* op3 state3,0 state3,1 state3,2 state3,3
*/
Map<OperatorInstanceID, List<OperatorStateHandle>> newManagedOperatorStates = reDistributePartitionableStates(
operatorStates,
newParallelism,
operatorIDs,
OperatorSubtaskState::getManagedOperatorState,
RoundRobinOperatorStateRepartitioner.INSTANCE);
Map<OperatorInstanceID, List<OperatorStateHandle>> newRawOperatorStates = reDistributePartitionableStates(
operatorStates,
newParallelism,
operatorIDs,
OperatorSubtaskState::getRawOperatorState,
RoundRobinOperatorStateRepartitioner.INSTANCE);
final Map<OperatorInstanceID, List<InputChannelStateHandle>> newInputChannelState = reDistributePartitionableStates(
operatorStates,
newParallelism,
operatorIDs,
OperatorSubtaskState::getInputChannelState,
channelStateNonRescalingRepartitioner("input channel"));
final Map<OperatorInstanceID, List<ResultSubpartitionStateHandle>> newResultSubpartitionState = reDistributePartitionableStates(
operatorStates,
newParallelism,
operatorIDs,
OperatorSubtaskState::getResultSubpartitionState,
channelStateNonRescalingRepartitioner("result subpartition"));
Map<OperatorInstanceID, List<KeyedStateHandle>> newManagedKeyedState = new HashMap<>(expectedNumberOfSubTasks);
Map<OperatorInstanceID, List<KeyedStateHandle>> newRawKeyedState = new HashMap<>(expectedNumberOfSubTasks);
reDistributeKeyedStates(
operatorStates,
newParallelism,
operatorIDs,
keyGroupPartitions,
newManagedKeyedState,
newRawKeyedState);
/*
* An executionJobVertex's all state handles needed to restore are something like a matrix
*
* parallelism0 parallelism1 parallelism2 parallelism3
* op0 sh(0,0) sh(0,1) sh(0,2) sh(0,3)
* op1 sh(1,0) sh(1,1) sh(1,2) sh(1,3)
* op2 sh(2,0) sh(2,1) sh(2,2) sh(2,3)
* op3 sh(3,0) sh(3,1) sh(3,2) sh(3,3)
*
*/
assignTaskStateToExecutionJobVertices(
executionJobVertex,
newManagedOperatorStates,
newRawOperatorStates,
newInputChannelState,
newResultSubpartitionState,
newManagedKeyedState,
newRawKeyedState,
newParallelism);
}
Example #23
| Source File: StateAssignmentOperation.java From flink with Apache License 2.0 | 4 votes |
|
private void assignTaskStateToExecutionJobVertices(
ExecutionJobVertex executionJobVertex,
Map<OperatorInstanceID, List<OperatorStateHandle>> subManagedOperatorState,
Map<OperatorInstanceID, List<OperatorStateHandle>> subRawOperatorState,
Map<OperatorInstanceID, List<InputChannelStateHandle>> inputChannelStates,
Map<OperatorInstanceID, List<ResultSubpartitionStateHandle>> resultSubpartitionStates,
Map<OperatorInstanceID, List<KeyedStateHandle>> subManagedKeyedState,
Map<OperatorInstanceID, List<KeyedStateHandle>> subRawKeyedState,
int newParallelism) {
List<OperatorIDPair> operatorIDs = executionJobVertex.getOperatorIDs();
for (int subTaskIndex = 0; subTaskIndex < newParallelism; subTaskIndex++) {
Execution currentExecutionAttempt = executionJobVertex.getTaskVertices()[subTaskIndex]
.getCurrentExecutionAttempt();
TaskStateSnapshot taskState = new TaskStateSnapshot(operatorIDs.size());
boolean statelessTask = true;
for (OperatorIDPair operatorID : operatorIDs) {
OperatorInstanceID instanceID = OperatorInstanceID.of(subTaskIndex, operatorID.getGeneratedOperatorID());
OperatorSubtaskState operatorSubtaskState = operatorSubtaskStateFrom(
instanceID,
subManagedOperatorState,
subRawOperatorState,
inputChannelStates,
resultSubpartitionStates,
subManagedKeyedState,
subRawKeyedState);
if (operatorSubtaskState.hasState()) {
statelessTask = false;
}
taskState.putSubtaskStateByOperatorID(operatorID.getGeneratedOperatorID(), operatorSubtaskState);
}
if (!statelessTask) {
JobManagerTaskRestore taskRestore = new JobManagerTaskRestore(restoreCheckpointId, taskState);
currentExecutionAttempt.setInitialState(taskRestore);
}
}
}
