org.apache.flink.runtime.state.FunctionSnapshotContext Java Examples

@Override
public void snapshotState(FunctionSnapshotContext context) throws Exception {

	counterPartitions.clear();

	checkCorrectSnapshot[getRuntimeContext().getIndexOfThisSubtask()] = counter;

	int div = counter / NUM_PARTITIONS;
	int mod = counter % NUM_PARTITIONS;

	for (int i = 0; i < NUM_PARTITIONS; ++i) {
		int partitionValue = div;
		if (mod > 0) {
			--mod;
			++partitionValue;
		}
		counterPartitions.add(partitionValue);
	}
}
 

@Override
public void snapshotState(FunctionSnapshotContext context) throws Exception {
	Preconditions.checkState(this.checkpointedState != null,
		"The " + getClass().getSimpleName() + " has not been properly initialized.");

	if (LOG.isDebugEnabled()) {
		LOG.debug("Checkpointing: Messages: {}, checkpoint id: {}, timestamp: {}",
			idsForCurrentCheckpoint, context.getCheckpointId(), context.getCheckpointTimestamp());
	}

	pendingCheckpoints.addLast(new Tuple2<>(context.getCheckpointId(), idsForCurrentCheckpoint));
	idsForCurrentCheckpoint = new HashSet<>(64);

	this.checkpointedState.clear();
	this.checkpointedState.add(SerializedCheckpointData.fromDeque(pendingCheckpoints, idSerializer));
}
 

@Override
public void snapshotState(FunctionSnapshotContext context) throws Exception {
	super.snapshotState(context);

	nextTransactionalIdHintState.clear();
	// To avoid duplication only first subtask keeps track of next transactional id hint. Otherwise all of the
	// subtasks would write exactly same information.
	if (getRuntimeContext().getIndexOfThisSubtask() == 0 && semantic == Semantic.EXACTLY_ONCE) {
		checkState(nextTransactionalIdHint != null, "nextTransactionalIdHint must be set for EXACTLY_ONCE");
		long nextFreeTransactionalId = nextTransactionalIdHint.nextFreeTransactionalId;

		// If we scaled up, some (unknown) subtask must have created new transactional ids from scratch. In that
		// case we adjust nextFreeTransactionalId by the range of transactionalIds that could be used for this
		// scaling up.
		if (getRuntimeContext().getNumberOfParallelSubtasks() > nextTransactionalIdHint.lastParallelism) {
			nextFreeTransactionalId += getRuntimeContext().getNumberOfParallelSubtasks() * kafkaProducersPoolSize;
		}

		nextTransactionalIdHintState.add(new NextTransactionalIdHint(
			getRuntimeContext().getNumberOfParallelSubtasks(),
			nextFreeTransactionalId));
	}
}
 

@Override
public void snapshotState(FunctionSnapshotContext context) throws Exception {
	bufferLock.lock();
	try {
		bufferState.clear();
		bufferState.addAll(buffer);

		offsetState.clear();
		offsetState.add(offset);

		uncompletedCheckpointMap.put(context.getCheckpointId(), offset);

		if (LOG.isDebugEnabled()) {
			LOG.debug("Checkpoint begin with checkpointId = " + context.getCheckpointId() +
				", lastCheckpointedOffset = " + lastCheckpointedOffset +
				", buffered results bytes = " + currentBufferBytes);
		}
	} finally {
		bufferLock.unlock();
	}
}
 

@Override
public void snapshotState(FunctionSnapshotContext context) throws Exception {
	final int numSubtasks = getRuntimeContext().getNumberOfParallelSubtasks();
	final int thisSubtaskIndex = getRuntimeContext().getIndexOfThisSubtask();

	// store total number of subtasks as broadcast state
	lastNumSubtasksBroadcastState.clear();
	lastNumSubtasksBroadcastState.put(LAST_NUM_SUBTASKS_STATE_KEY, numSubtasks);

	// populate broadcast state (identical across all subtasks)
	broadcastElementsState.clear();
	for (int i = 0; i < numSubtasks; i++) {
		broadcastElementsState.put(i, getBroadcastStateEntryValue(i));
	}

	// each subtask only stores its own subtask index as a subset of the union set
	unionElementsState.clear();
	unionElementsState.add(thisSubtaskIndex);
}
 

@Override
public void snapshotState(FunctionSnapshotContext context) throws Exception {
	if (stateRecorded.get()) {
		minimalCheckpointIdIncludingData.compareAndSet(Long.MAX_VALUE,
			context.getCheckpointId());
	}
	if (checkpointCompletedIncludingData.get()) {
		// there is a checkpoint completed with state data, we can trigger the failure now
		waitForCheckpointLatch.trigger();
		failInCheckpointLatch.await();
		if (!failedAlready.getAndSet(true)) {
			throw new RuntimeException("Failing on purpose.");
		} else {
			// make sure there would be no more successful checkpoint before job failing
			// otherwise there might be an unexpected successful checkpoint even
			// CheckpointFailureManager has decided to fail the job
			blockSnapshotLatch.await();
		}
	}
}
 

@Override
public void snapshotState(FunctionSnapshotContext context) throws Exception {
	final int numSubtasks = getRuntimeContext().getNumberOfParallelSubtasks();
	final int thisSubtaskIndex = getRuntimeContext().getIndexOfThisSubtask();

	// store total number of subtasks as broadcast state
	lastNumSubtasksBroadcastState.clear();
	lastNumSubtasksBroadcastState.put(LAST_NUM_SUBTASKS_STATE_KEY, numSubtasks);

	// populate broadcast state (identical across all subtasks)
	broadcastElementsState.clear();
	for (int i = 0; i < numSubtasks; i++) {
		broadcastElementsState.put(i, getBroadcastStateEntryValue(i));
	}

	// each subtask only stores its own subtask index as a subset of the union set
	unionElementsState.clear();
	unionElementsState.add(thisSubtaskIndex);
}
 

@Override
public void snapshotState(FunctionSnapshotContext context) throws Exception {
    // check for asynchronous errors and fail the checkpoint if necessary
    checkErroneous();

    if (flushOnCheckpoint) {
        // wait until all the messages are acknowledged
        synchronized (pendingRecordsLock) {
            while (pendingRecords > 0) {
                pendingRecordsLock.wait(100);
            }
        }

        // if the flushed requests has errors, we should propagate it also and fail the checkpoint
        checkErroneous();
    }
}
 

@Override
public void snapshotState(FunctionSnapshotContext context) throws Exception {
	// this is like the pre-commit of a 2-phase-commit transaction
	// we are ready to commit and remember the transaction

	checkState(currentTransactionHolder != null, "bug: no transaction object when performing state snapshot");

	long checkpointId = context.getCheckpointId();
	LOG.debug("{} - checkpoint {} triggered, flushing transaction '{}'", name(), context.getCheckpointId(), currentTransactionHolder);

	preCommit(currentTransactionHolder.handle);
	pendingCommitTransactions.put(checkpointId, currentTransactionHolder);
	LOG.debug("{} - stored pending transactions {}", name(), pendingCommitTransactions);

	currentTransactionHolder = beginTransactionInternal();
	LOG.debug("{} - started new transaction '{}'", name(), currentTransactionHolder);

	state.clear();
	state.add(new State<>(
		this.currentTransactionHolder,
		new ArrayList<>(pendingCommitTransactions.values()),
		userContext));
}
 

@Override
public void snapshotState(FunctionSnapshotContext context) throws Exception {
	Preconditions.checkState(this.checkpointedState != null,
		"The " + getClass().getSimpleName() + " has not been properly initialized.");

	if (LOG.isDebugEnabled()) {
		LOG.debug("{} checkpointing: Messages: {}, checkpoint id: {}, timestamp: {}",
			idsForCurrentCheckpoint, context.getCheckpointId(), context.getCheckpointTimestamp());
	}

	pendingCheckpoints.addLast(new Tuple2<>(context.getCheckpointId(), idsForCurrentCheckpoint));
	idsForCurrentCheckpoint = new HashSet<>(64);

	this.checkpointedState.clear();
	this.checkpointedState.add(SerializedCheckpointData.fromDeque(pendingCheckpoints, idSerializer));
}
 

@Override
public void snapshotState(FunctionSnapshotContext ctx) throws Exception {
	// check for asynchronous errors and fail the checkpoint if necessary
	checkErroneous();

	if (flushOnCheckpoint) {
		// flushing is activated: We need to wait until pendingRecords is 0
		flush();
		synchronized (pendingRecordsLock) {
			if (pendingRecords != 0) {
				throw new IllegalStateException("Pending record count must be zero at this point: " + pendingRecords);
			}

			// if the flushed requests has errors, we should propagate it also and fail the checkpoint
			checkErroneous();
		}
	}
}
 

@Override
public void snapshotState(FunctionSnapshotContext context) throws Exception {
	// this is like the pre-commit of a 2-phase-commit transaction
	// we are ready to commit and remember the transaction

	checkState(currentTransactionHolder != null, "bug: no transaction object when performing state snapshot");

	long checkpointId = context.getCheckpointId();
	LOG.debug("{} - checkpoint {} triggered, flushing transaction '{}'", name(), context.getCheckpointId(), currentTransactionHolder);

	preCommit(currentTransactionHolder.handle);
	pendingCommitTransactions.put(checkpointId, currentTransactionHolder);
	LOG.debug("{} - stored pending transactions {}", name(), pendingCommitTransactions);

	currentTransactionHolder = beginTransactionInternal();
	LOG.debug("{} - started new transaction '{}'", name(), currentTransactionHolder);

	state.clear();
	state.add(new State<>(
		this.currentTransactionHolder,
		new ArrayList<>(pendingCommitTransactions.values()),
		userContext));
}
 

@Override
public void snapshotState(FunctionSnapshotContext context) throws Exception {
	super.snapshotState(context);

	nextTransactionalIdHintState.clear();
	// To avoid duplication only first subtask keeps track of next transactional id hint. Otherwise all of the
	// subtasks would write exactly same information.
	if (getRuntimeContext().getIndexOfThisSubtask() == 0 && semantic == Semantic.EXACTLY_ONCE) {
		checkState(nextTransactionalIdHint != null, "nextTransactionalIdHint must be set for EXACTLY_ONCE");
		long nextFreeTransactionalId = nextTransactionalIdHint.nextFreeTransactionalId;

		// If we scaled up, some (unknown) subtask must have created new transactional ids from scratch. In that
		// case we adjust nextFreeTransactionalId by the range of transactionalIds that could be used for this
		// scaling up.
		if (getRuntimeContext().getNumberOfParallelSubtasks() > nextTransactionalIdHint.lastParallelism) {
			nextFreeTransactionalId += getRuntimeContext().getNumberOfParallelSubtasks() * kafkaProducersPoolSize;
		}

		nextTransactionalIdHintState.add(new NextTransactionalIdHint(
			getRuntimeContext().getNumberOfParallelSubtasks(),
			nextFreeTransactionalId));
	}
}
 

@Override
public void snapshotState(FunctionSnapshotContext context) throws Exception {
	super.snapshotState(context);

	nextTransactionalIdHintState.clear();
	// To avoid duplication only first subtask keeps track of next transactional id hint. Otherwise all of the
	// subtasks would write exactly same information.
	if (getRuntimeContext().getIndexOfThisSubtask() == 0 && semantic == FlinkKafkaProducer.Semantic.EXACTLY_ONCE) {
		checkState(nextTransactionalIdHint != null, "nextTransactionalIdHint must be set for EXACTLY_ONCE");
		long nextFreeTransactionalId = nextTransactionalIdHint.nextFreeTransactionalId;

		// If we scaled up, some (unknown) subtask must have created new transactional ids from scratch. In that
		// case we adjust nextFreeTransactionalId by the range of transactionalIds that could be used for this
		// scaling up.
		if (getRuntimeContext().getNumberOfParallelSubtasks() > nextTransactionalIdHint.lastParallelism) {
			nextFreeTransactionalId += getRuntimeContext().getNumberOfParallelSubtasks() * kafkaProducersPoolSize;
		}

		nextTransactionalIdHintState.add(new FlinkKafkaProducer.NextTransactionalIdHint(
			getRuntimeContext().getNumberOfParallelSubtasks(),
			nextFreeTransactionalId));
	}
}
 

@Override
public void snapshotState(FunctionSnapshotContext context) throws Exception {
	unionListState.clear();
	unionListState.add(CHECKPOINTED_STRING);
	unionListState.add(CHECKPOINTED_STRING_1);
	unionListState.add(CHECKPOINTED_STRING_2);
	unionListState.add(CHECKPOINTED_STRING_3);
}
 

@Override
public void snapshotState(FunctionSnapshotContext context) throws Exception {
	Iterator<Map.Entry<BaseRow, Map<BaseRow, RankRow>>> iter = kvRowKeyMap.entrySet().iterator();
	while (iter.hasNext()) {
		Map.Entry<BaseRow, Map<BaseRow, RankRow>> entry = iter.next();
		BaseRow partitionKey = entry.getKey();
		Map<BaseRow, RankRow> currentRowKeyMap = entry.getValue();
		keyContext.setCurrentKey(partitionKey);
		flushBufferToState(currentRowKeyMap);
	}
}
 

@Override
public void snapshotState(FunctionSnapshotContext functionSnapshotContext) throws Exception {
    // 将 buffer 中的数据保存到状态中，来保证 Exactly Once
    localPvStatListState.clear();
    for (Map.Entry<String, Long> appIdPv : localPvStat.entrySet()) {
        localPvStatListState.add(Tuple2.of(appIdPv.getKey(), appIdPv.getValue()));
        log.info("snapshot   subtask: {}    appId: {}   pv: {}", subtaskIndex, appIdPv.getKey(), appIdPv.getValue());
    }
}
 

@Override
public void snapshotState(FunctionSnapshotContext context) throws Exception {
	Preconditions.checkNotNull(restoredBucketStates, "The operator has not been properly initialized.");

	restoredBucketStates.clear();

	synchronized (state.bucketStates) {
		int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask();

		for (Map.Entry<String, BucketState<T>> bucketStateEntry : state.bucketStates.entrySet()) {
			BucketState<T> bucketState = bucketStateEntry.getValue();

			if (bucketState.isWriterOpen) {
				bucketState.currentFileValidLength = bucketState.writer.flush();
			}

			synchronized (bucketState.pendingFilesPerCheckpoint) {
				bucketState.pendingFilesPerCheckpoint.put(context.getCheckpointId(), bucketState.pendingFiles);
			}
			bucketState.pendingFiles = new ArrayList<>();
		}
		restoredBucketStates.add(state);

		if (LOG.isDebugEnabled()) {
			LOG.debug("{} idx {} checkpointed {}.", getClass().getSimpleName(), subtaskIdx, state);
		}
	}
}
 

@Override
public void snapshotState(FunctionSnapshotContext context)
        throws Exception
{
    realTimeSink.flush();
    //unionState.add(thisState);
}
 

@Override
public void snapshotState(FunctionSnapshotContext context) throws Exception {
    // called when a snapshot for a checkpoint is requested

    if (!runningChecker.isRunning()) {
        LOG.debug("snapshotState() called on closed source; returning null.");
        return;
    }

    if (LOG.isDebugEnabled()) {
        LOG.debug("Snapshotting state {} ...", context.getCheckpointId());
    }

    unionOffsetStates.clear();

    if (LOG.isDebugEnabled()) {
        LOG.debug("Snapshotted state, last processed offsets: {}, checkpoint id: {}, timestamp: {}",
                offsetTable, context.getCheckpointId(), context.getCheckpointTimestamp());
    }

    // remove the unassigned queues in order to avoid read the wrong offset when the source restart
    Set<MessageQueue> assignedQueues = consumer.fetchMessageQueuesInBalance(topic);
    offsetTable.entrySet().removeIf(item -> !assignedQueues.contains(item.getKey()));

    for (Map.Entry<MessageQueue, Long> entry : offsetTable.entrySet()) {
        unionOffsetStates.add(Tuple2.of(entry.getKey(), entry.getValue()));
    }
}
 

@Override
public void snapshotState(FunctionSnapshotContext context) throws Exception {
    if (LOG.isDebugEnabled()) {
        LOG.debug("Snapshotting state {} ...", context.getCheckpointId());
    }
    this.connector.flush();
}
 

@Override
public void snapshotState(FunctionSnapshotContext context) throws Exception {
	Preconditions.checkState(bucketStates != null && maxPartCountersState != null, "sink has not been initialized");

	buckets.snapshotState(
			context.getCheckpointId(),
			bucketStates,
			maxPartCountersState);
}
 

@Override
public void snapshotState(FunctionSnapshotContext context) throws Exception {
	//before checkpoints make sure all the batched / buffered pubsub messages have actually been sent
	publisher.publishAllOutstanding();

	// At this point, no new messages will be published because this thread has successfully acquired
	// the checkpoint lock. So we just wait for all the pending futures to complete.
	waitForFuturesToComplete();
	if (exceptionAtomicReference.get() != null) {
		throw exceptionAtomicReference.get();
	}
}
 

@Override
public void snapshotState(FunctionSnapshotContext context) throws Exception {
	// check for asynchronous errors and fail the checkpoint if necessary
	checkAndPropagateAsyncError();

	flushSync();
	if (producer.getOutstandingRecordsCount() > 0) {
		throw new IllegalStateException(
			"Number of outstanding records must be zero at this point: " + producer.getOutstandingRecordsCount());
	}

	// if the flushed requests has errors, we should propagate it also and fail the checkpoint
	checkAndPropagateAsyncError();
}
 

@Override
public void snapshotState(FunctionSnapshotContext context) throws Exception {
    isFlushed = false;
    super.snapshotState(context);
    // if the snapshot implementation doesn't wait until
    // all pending records are flushed, we should fail the test
    if (flushOnCheckpoint && !isFlushed) {
        throw new RuntimeException("Flushing is enabled; snapshots should be blocked" +
                " until all pending records are flushed");
    }
}
 

@Override
public void snapshotState(FunctionSnapshotContext context) throws Exception {
	Preconditions.checkState(this.checkpointedState != null,
		"The " + getClass().getSimpleName() + " has not been properly initialized.");

	this.checkpointedState.clear();
	this.checkpointedState.add(this.numElementsEmitted);
}
 

@Override
public void snapshotState(FunctionSnapshotContext context) throws Exception {
	Preconditions.checkState(this.checkpointedState != null,
		"The " + getClass().getSimpleName() + " state has not been properly initialized.");

	this.checkpointedState.clear();
	this.checkpointedState.add(this.globalModificationTime);

	if (LOG.isDebugEnabled()) {
		LOG.debug("{} checkpointed {}.", getClass().getSimpleName(), globalModificationTime);
	}
}
 

@Override
public void snapshotState(FunctionSnapshotContext context) throws Exception {
	Preconditions.checkState(this.checkpointedState != null,
		"The " + getClass().getSimpleName() + " has not been properly initialized.");

	this.checkpointedState.clear();
	this.checkpointedState.add(this.numElementsEmitted);
}
 

@Override
public void snapshotState(FunctionSnapshotContext context) throws Exception {
	Preconditions.checkState(this.checkpointedState != null,
		"The " + getClass().getSimpleName() + " has not been properly initialized.");

	this.checkpointedState.clear();
	this.checkpointedState.add(this.numElementsEmitted);
}
 

@Override
public void snapshotState(FunctionSnapshotContext context) throws Exception {
    Preconditions.checkState(this.checkpointedState != null,
            "The " + getClass().getSimpleName() + " has not been properly initialized.");

    this.checkpointedState.clear();
    this.checkpointedState.add(this.numElementsEmitted);
}