org.apache.flink.runtime.state.StateSnapshotContext Java Examples

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	super.snapshotState(context);

	ListState<StreamElement> partitionableState =
		getOperatorStateBackend().getListState(new ListStateDescriptor<>(STATE_NAME, inStreamElementSerializer));
	partitionableState.clear();

	Collection<StreamElementQueueEntry<?>> values = queue.values();

	try {
		for (StreamElementQueueEntry<?> value : values) {
			partitionableState.add(value.getStreamElement());
		}

		// add the pending stream element queue entry if the stream element queue is currently full
		if (pendingStreamElementQueueEntry != null) {
			partitionableState.add(pendingStreamElementQueueEntry.getStreamElement());
		}
	} catch (Exception e) {
		partitionableState.clear();

		throw new Exception("Could not add stream element queue entries to operator state " +
			"backend of operator " + getOperatorName() + '.', e);
	}
}
 

public static void snapshotFunctionState(
		StateSnapshotContext context,
		OperatorStateBackend backend,
		Function userFunction) throws Exception {

	Preconditions.checkNotNull(context);
	Preconditions.checkNotNull(backend);

	while (true) {

		if (trySnapshotFunctionState(context, backend, userFunction)) {
			break;
		}

		// inspect if the user function is wrapped, then unwrap and try again if we can snapshot the inner function
		if (userFunction instanceof WrappingFunction) {
			userFunction = ((WrappingFunction<?>) userFunction).getWrappedFunction();
		} else {
			break;
		}
	}
}
 

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	super.snapshotState(context);

	ListState<StreamElement> partitionableState =
		getOperatorStateBackend().getListState(new ListStateDescriptor<>(STATE_NAME, inStreamElementSerializer));
	partitionableState.clear();

	try {
		partitionableState.addAll(queue.values());
	} catch (Exception e) {
		partitionableState.clear();

		throw new Exception("Could not add stream element queue entries to operator state " +
			"backend of operator " + getOperatorName() + '.', e);
	}
}
 

public static void snapshotFunctionState(
		StateSnapshotContext context,
		OperatorStateBackend backend,
		Function userFunction) throws Exception {

	Preconditions.checkNotNull(context);
	Preconditions.checkNotNull(backend);

	while (true) {

		if (trySnapshotFunctionState(context, backend, userFunction)) {
			break;
		}

		// inspect if the user function is wrapped, then unwrap and try again if we can snapshot the inner function
		if (userFunction instanceof WrappingFunction) {
			userFunction = ((WrappingFunction<?>) userFunction).getWrappedFunction();
		} else {
			break;
		}
	}
}
 

public static void snapshotFunctionState(
		StateSnapshotContext context,
		OperatorStateBackend backend,
		Function userFunction) throws Exception {

	Preconditions.checkNotNull(context);
	Preconditions.checkNotNull(backend);

	while (true) {

		if (trySnapshotFunctionState(context, backend, userFunction)) {
			break;
		}

		// inspect if the user function is wrapped, then unwrap and try again if we can snapshot the inner function
		if (userFunction instanceof WrappingFunction) {
			userFunction = ((WrappingFunction<?>) userFunction).getWrappedFunction();
		} else {
			break;
		}
	}
}
 

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	super.snapshotState(context);

	ListState<StreamElement> partitionableState =
		getOperatorStateBackend().getListState(new ListStateDescriptor<>(STATE_NAME, inStreamElementSerializer));
	partitionableState.clear();

	Collection<StreamElementQueueEntry<?>> values = queue.values();

	try {
		for (StreamElementQueueEntry<?> value : values) {
			partitionableState.add(value.getStreamElement());
		}

		// add the pending stream element queue entry if the stream element queue is currently full
		if (pendingStreamElementQueueEntry != null) {
			partitionableState.add(pendingStreamElementQueueEntry.getStreamElement());
		}
	} catch (Exception e) {
		partitionableState.clear();

		throw new Exception("Could not add stream element queue entries to operator state " +
			"backend of operator " + getOperatorName() + '.', e);
	}
}
 

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	super.snapshotState(context);

	checkState(checkpointedState != null,
		"The operator state has not been properly initialized.");

	int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask();

	checkpointedState.clear();

	List<TimestampedFileInputSplit> readerState = reader.getReaderState();

	try {
		for (TimestampedFileInputSplit split : readerState) {
			// create a new partition for each entry.
			checkpointedState.add(split);
		}
	} catch (Exception e) {
		checkpointedState.clear();

		throw new Exception("Could not add timestamped file input splits to to operator " +
			"state backend of operator " + getOperatorName() + '.', e);
	}

	if (LOG.isDebugEnabled()) {
		LOG.debug("{} (taskIdx={}) checkpointed {} splits: {}.",
			getClass().getSimpleName(), subtaskIdx, readerState.size(), readerState);
	}
}
 

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	super.snapshotState(context);
	// clear state first
	bufferState.clear();

	List<Tuple2<RowData, Long>> dataToFlush = new ArrayList<>(inputBuffer.size());
	inputBuffer.forEach((key, value) -> dataToFlush.add(Tuple2.of(key, value)));

	// batch put
	bufferState.addAll(dataToFlush);
}
 

@Override
public final void snapshotState(StateSnapshotContext context) throws Exception {
  if (checkpointStats != null) {
    checkpointStats.snapshotStart(context.getCheckpointId());
  }

  if (requiresStableInput) {
    // We notify the BufferingDoFnRunner to associate buffered state with this
    // snapshot id and start a new buffer for elements arriving after this snapshot.
    bufferingDoFnRunner.checkpoint(context.getCheckpointId());
  }

  try {
    outputManager.openBuffer();
    // Ensure that no new bundle gets started as part of finishing a bundle
    while (bundleStarted) {
      invokeFinishBundle();
    }
    outputManager.closeBuffer();
  } catch (Exception e) {
    // https://jira.apache.org/jira/browse/FLINK-14653
    // Any regular exception during checkpointing will be tolerated by Flink because those
    // typically do not affect the execution flow. We need to fail hard here because errors
    // in bundle execution are application errors which are not related to checkpointing.
    throw new Error("Checkpointing failed because bundle failed to finalize.", e);
  }

  super.snapshotState(context);
}
 

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	if (context.getCheckpointId() == DECLINE_CHECKPOINT_ID) {
		DeclineSink.waitLatch.await();
	}
	super.snapshotState(context);
}
 

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	OperatorStateCheckpointOutputStream outStream = context.getRawOperatorStateOutput();

	IN_CHECKPOINT_LATCH.trigger();

	// this should lock
	outStream.write(1);
}
 

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	ListState<Integer> partitionableState =
		getOperatorStateBackend().getListState(TEST_DESCRIPTOR);
	partitionableState.clear();

	partitionableState.add(42);
	partitionableState.add(4711);

	++numberSnapshotCalls;
}
 

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	super.snapshotState(context);

	checkState(checkpointedState != null,
		"The operator state has not been properly initialized.");

	int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask();

	checkpointedState.clear();

	List<T> readerState = getReaderState();

	try {
		for (T split : readerState) {
			checkpointedState.add(split);
		}
	} catch (Exception e) {
		checkpointedState.clear();

		throw new Exception("Could not add timestamped file input splits to to operator " +
				"state backend of operator " + getOperatorName() + '.', e);
	}

	if (LOG.isDebugEnabled()) {
		LOG.debug("{} (taskIdx={}) checkpointed {} splits: {}.",
				getClass().getSimpleName(), subtaskIdx, readerState.size(), readerState);
	}
}
 

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	super.snapshotState(context);

	Preconditions.checkState(this.checkpointedState != null,
		"The operator state has not been properly initialized.");

	saveHandleInState(context.getCheckpointId(), context.getCheckpointTimestamp());

	this.checkpointedState.clear();

	try {
		for (PendingCheckpoint pendingCheckpoint : pendingCheckpoints) {
			// create a new partition for each entry.
			this.checkpointedState.add(pendingCheckpoint);
		}
	} catch (Exception e) {
		checkpointedState.clear();

		throw new Exception("Could not add panding checkpoints to operator state " +
			"backend of operator " + getOperatorName() + '.', e);
	}

	int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask();
	if (LOG.isDebugEnabled()) {
		LOG.debug("{} (taskIdx= {}) checkpointed {}.", getClass().getSimpleName(), subtaskIdx, this.pendingCheckpoints);
	}
}
 

private static boolean trySnapshotFunctionState(
		StateSnapshotContext context,
		OperatorStateBackend backend,
		Function userFunction) throws Exception {

	if (userFunction instanceof CheckpointedFunction) {
		((CheckpointedFunction) userFunction).snapshotState(context);

		return true;
	}

	if (userFunction instanceof ListCheckpointed) {
		@SuppressWarnings("unchecked")
		List<Serializable> partitionableState = ((ListCheckpointed<Serializable>) userFunction).
				snapshotState(context.getCheckpointId(), context.getCheckpointTimestamp());

		ListState<Serializable> listState = backend.
				getSerializableListState(DefaultOperatorStateBackend.DEFAULT_OPERATOR_STATE_NAME);

		listState.clear();

		if (null != partitionableState) {
			try {
				for (Serializable statePartition : partitionableState) {
					listState.add(statePartition);
				}
			} catch (Exception e) {
				listState.clear();

				throw new Exception("Could not write partitionable state to operator " +
					"state backend.", e);
			}
		}

		return true;
	}

	return false;
}
 

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	super.snapshotState(context);

	Preconditions.checkState(this.checkpointedState != null,
		"The operator state has not been properly initialized.");

	saveHandleInState(context.getCheckpointId(), context.getCheckpointTimestamp());

	this.checkpointedState.clear();

	try {
		for (PendingCheckpoint pendingCheckpoint : pendingCheckpoints) {
			// create a new partition for each entry.
			this.checkpointedState.add(pendingCheckpoint);
		}
	} catch (Exception e) {
		checkpointedState.clear();

		throw new Exception("Could not add panding checkpoints to operator state " +
			"backend of operator " + getOperatorName() + '.', e);
	}

	int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask();
	if (LOG.isDebugEnabled()) {
		LOG.debug("{} (taskIdx= {}) checkpointed {}.", getClass().getSimpleName(), subtaskIdx, this.pendingCheckpoints);
	}
}
 

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	super.snapshotState(context);
	// clear state first
	bufferState.clear();

	List<Tuple2<BaseRow, Long>> dataToFlush = new ArrayList<>(inputBuffer.size());
	inputBuffer.forEach((key, value) -> dataToFlush.add(Tuple2.of(key, value)));

	// batch put
	bufferState.addAll(dataToFlush);
}
 

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	OperatorStateCheckpointOutputStream outStream = context.getRawOperatorStateOutput();

	IN_CHECKPOINT_LATCH.trigger();

	// this should lock
	outStream.write(1);
}
 

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	ListState<Integer> partitionableState =
		getOperatorStateBackend().getListState(TEST_DESCRIPTOR);
	partitionableState.clear();

	partitionableState.add(42);
	partitionableState.add(4711);

	++numberSnapshotCalls;
}
 

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	ListState<Integer> partitionableState =
		getOperatorStateBackend().getListState(TEST_DESCRIPTOR);
	partitionableState.clear();

	partitionableState.add(42);
	partitionableState.add(4711);

	++numberSnapshotCalls;
}
 

private static boolean trySnapshotFunctionState(
		StateSnapshotContext context,
		OperatorStateBackend backend,
		Function userFunction) throws Exception {

	if (userFunction instanceof CheckpointedFunction) {
		((CheckpointedFunction) userFunction).snapshotState(context);

		return true;
	}

	if (userFunction instanceof ListCheckpointed) {
		@SuppressWarnings("unchecked")
		List<Serializable> partitionableState = ((ListCheckpointed<Serializable>) userFunction).
				snapshotState(context.getCheckpointId(), context.getCheckpointTimestamp());

		ListState<Serializable> listState = backend.
				getSerializableListState(DefaultOperatorStateBackend.DEFAULT_OPERATOR_STATE_NAME);

		listState.clear();

		if (null != partitionableState) {
			try {
				for (Serializable statePartition : partitionableState) {
					listState.add(statePartition);
				}
			} catch (Exception e) {
				listState.clear();

				throw new Exception("Could not write partitionable state to operator " +
					"state backend.", e);
			}
		}

		return true;
	}

	return false;
}
 

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	super.snapshotState(context);

	Preconditions.checkState(this.checkpointedState != null,
		"The operator state has not been properly initialized.");

	saveHandleInState(context.getCheckpointId(), context.getCheckpointTimestamp());

	this.checkpointedState.clear();

	try {
		for (PendingCheckpoint pendingCheckpoint : pendingCheckpoints) {
			// create a new partition for each entry.
			this.checkpointedState.add(pendingCheckpoint);
		}
	} catch (Exception e) {
		checkpointedState.clear();

		throw new Exception("Could not add panding checkpoints to operator state " +
			"backend of operator " + getOperatorName() + '.', e);
	}

	int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask();
	if (LOG.isDebugEnabled()) {
		LOG.debug("{} (taskIdx= {}) checkpointed {}.", getClass().getSimpleName(), subtaskIdx, this.pendingCheckpoints);
	}
}
 

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	super.snapshotState(context);

	checkState(checkpointedState != null,
		"The operator state has not been properly initialized.");

	int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask();

	checkpointedState.clear();

	List<TimestampedFileInputSplit> readerState = reader.getReaderState();

	try {
		for (TimestampedFileInputSplit split : readerState) {
			// create a new partition for each entry.
			checkpointedState.add(split);
		}
	} catch (Exception e) {
		checkpointedState.clear();

		throw new Exception("Could not add timestamped file input splits to to operator " +
			"state backend of operator " + getOperatorName() + '.', e);
	}

	if (LOG.isDebugEnabled()) {
		LOG.debug("{} (taskIdx={}) checkpointed {} splits: {}.",
			getClass().getSimpleName(), subtaskIdx, readerState.size(), readerState);
	}
}
 

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	OperatorStateCheckpointOutputStream outStream = context.getRawOperatorStateOutput();

	IN_CHECKPOINT_LATCH.trigger();

	// this should lock
	outStream.write(1);
}
 

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
  super.snapshotState(context);
  this.feedbackLogger.startLogging(context.getRawKeyedOperatorStateOutput());
}
 

private static boolean trySnapshotFunctionState(
		StateSnapshotContext context,
		OperatorStateBackend backend,
		Function userFunction) throws Exception {

	if (userFunction instanceof CheckpointedFunction) {
		((CheckpointedFunction) userFunction).snapshotState(context);

		return true;
	}

	if (userFunction instanceof ListCheckpointed) {
		@SuppressWarnings("unchecked")
		List<Serializable> partitionableState = ((ListCheckpointed<Serializable>) userFunction).
				snapshotState(context.getCheckpointId(), context.getCheckpointTimestamp());

		// We are using JavaSerializer from the flink-runtime module here. This is very naughty and
		// we shouldn't be doing it because ideally nothing in the API modules/connector depends
		// directly on flink-runtime. We are doing it here because we need to maintain backwards
		// compatibility with old state and because we will have to rework/remove this code soon.
		ListStateDescriptor<Serializable> listStateDescriptor =
			new ListStateDescriptor<>(DefaultOperatorStateBackend.DEFAULT_OPERATOR_STATE_NAME, new JavaSerializer<>());
		ListState<Serializable> listState = backend.getListState(listStateDescriptor);

		listState.clear();

		if (null != partitionableState) {
			try {
				for (Serializable statePartition : partitionableState) {
					listState.add(statePartition);
				}
			} catch (Exception e) {
				listState.clear();

				throw new Exception("Could not write partitionable state to operator " +
					"state backend.", e);
			}
		}

		return true;
	}

	return false;
}
 

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	ACTUAL_ORDER_TRACKING.add("OPERATOR::snapshotState");
	super.snapshotState(context);
}
 

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	counterState.add(counter);
}
 

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
}
 

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	LOG.debug("Taking a snapshot for checkpoint {}", context.getCheckpointId());
	readerState.update(sourceReader.snapshotState());
}