Java Code Examples for org.apache.flink.runtime.state.FunctionSnapshotContext#getCheckpointId()

The following examples show how to use org.apache.flink.runtime.state.FunctionSnapshotContext#getCheckpointId() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: TwoPhaseCommitSinkFunction.java From Flink-CEPplus with Apache License 2.0

6 votes

@Override
public void snapshotState(FunctionSnapshotContext context) throws Exception {
	// this is like the pre-commit of a 2-phase-commit transaction
	// we are ready to commit and remember the transaction

	checkState(currentTransactionHolder != null, "bug: no transaction object when performing state snapshot");

	long checkpointId = context.getCheckpointId();
	LOG.debug("{} - checkpoint {} triggered, flushing transaction '{}'", name(), context.getCheckpointId(), currentTransactionHolder);

	preCommit(currentTransactionHolder.handle);
	pendingCommitTransactions.put(checkpointId, currentTransactionHolder);
	LOG.debug("{} - stored pending transactions {}", name(), pendingCommitTransactions);

	currentTransactionHolder = beginTransactionInternal();
	LOG.debug("{} - started new transaction '{}'", name(), currentTransactionHolder);

	state.clear();
	state.add(new State<>(
		this.currentTransactionHolder,
		new ArrayList<>(pendingCommitTransactions.values()),
		userContext));
}

Example 2

Source File: TwoPhaseCommitSinkFunction.java From flink with Apache License 2.0

6 votes

@Override
public void snapshotState(FunctionSnapshotContext context) throws Exception {
	// this is like the pre-commit of a 2-phase-commit transaction
	// we are ready to commit and remember the transaction

	checkState(currentTransactionHolder != null, "bug: no transaction object when performing state snapshot");

	long checkpointId = context.getCheckpointId();
	LOG.debug("{} - checkpoint {} triggered, flushing transaction '{}'", name(), context.getCheckpointId(), currentTransactionHolder);

	preCommit(currentTransactionHolder.handle);
	pendingCommitTransactions.put(checkpointId, currentTransactionHolder);
	LOG.debug("{} - stored pending transactions {}", name(), pendingCommitTransactions);

	currentTransactionHolder = beginTransactionInternal();
	LOG.debug("{} - started new transaction '{}'", name(), currentTransactionHolder);

	state.clear();
	state.add(new State<>(
		this.currentTransactionHolder,
		new ArrayList<>(pendingCommitTransactions.values()),
		userContext));
}

Example 3

Source File: TwoPhaseCommitSinkFunction.java From flink with Apache License 2.0

6 votes

@Override
public void snapshotState(FunctionSnapshotContext context) throws Exception {
	// this is like the pre-commit of a 2-phase-commit transaction
	// we are ready to commit and remember the transaction

	checkState(currentTransactionHolder != null, "bug: no transaction object when performing state snapshot");

	long checkpointId = context.getCheckpointId();
	LOG.debug("{} - checkpoint {} triggered, flushing transaction '{}'", name(), context.getCheckpointId(), currentTransactionHolder);

	preCommit(currentTransactionHolder.handle);
	pendingCommitTransactions.put(checkpointId, currentTransactionHolder);
	LOG.debug("{} - stored pending transactions {}", name(), pendingCommitTransactions);

	currentTransactionHolder = beginTransactionInternal();
	LOG.debug("{} - started new transaction '{}'", name(), currentTransactionHolder);

	state.clear();
	state.add(new State<>(
		this.currentTransactionHolder,
		new ArrayList<>(pendingCommitTransactions.values()),
		userContext));
}

Example 4

Source File: ZooKeeperHighAvailabilityITCase.java From Flink-CEPplus with Apache License 2.0

5 votes

@Override
public void snapshotState(FunctionSnapshotContext context) throws Exception {
	if (context.getCheckpointId() > 5) {
		waitForCheckpointLatch.trigger();
		failInCheckpointLatch.await();
		if (!failedAlready.getAndSet(true)) {
			throw new RuntimeException("Failing on purpose.");
		}
	}
}

Example 5

Source File: FailingCollectionSource.java From flink with Apache License 2.0

5 votes

@Override
public void snapshotState(FunctionSnapshotContext context) throws Exception {
	Preconditions.checkState(
		this.checkpointedState != null,
		"The " + getClass().getSimpleName() + " has not been properly initialized.");

	this.checkpointedState.clear();
	this.checkpointedState.add(this.numElementsEmitted);
	long checkpointId = context.getCheckpointId();
	checkpointedEmittedNums.put(checkpointId, numElementsEmitted);
}

Example 6

Source File: ZooKeeperHighAvailabilityITCase.java From flink with Apache License 2.0

5 votes

@Override
public void snapshotState(FunctionSnapshotContext context) throws Exception {
	if (context.getCheckpointId() > 5) {
		waitForCheckpointLatch.trigger();
		failInCheckpointLatch.await();
		if (!failedAlready.getAndSet(true)) {
			throw new RuntimeException("Failing on purpose.");
		} else {
			// make sure there would be no more successful checkpoint before job failing
			// otherwise there might be a successful checkpoint 7 which is unexpected
			// we expect checkpoint 5 is the last successful one before ha storage recovering
			blockSnapshotLatch.await();
		}
	}
}

Example 7

Source File: FailingCollectionSource.java From flink with Apache License 2.0

5 votes

@Override
public void snapshotState(FunctionSnapshotContext context) throws Exception {
	Preconditions.checkState(
		this.checkpointedState != null,
		"The " + getClass().getSimpleName() + " has not been properly initialized.");

	this.checkpointedState.clear();
	this.checkpointedState.add(this.numElementsEmitted);
	long checkpointId = context.getCheckpointId();
	checkpointedEmittedNums.put(checkpointId, numElementsEmitted);
}

Example 8

Source File: CancellingIntegerSource.java From flink with Apache License 2.0

5 votes

@Override
public void snapshotState(FunctionSnapshotContext context) throws Exception {
	lastSentStored.update(singletonList(sentCount));
	if (cancelAfter != null && cancelAfter <= sentCount && cancelAfterCheckpointId == null) {
		cancelAfterCheckpointId = context.getCheckpointId();
	}
}

Example 9

Source File: UnboundedSourceWrapper.java From beam with Apache License 2.0

4 votes

@Override
public void snapshotState(FunctionSnapshotContext functionSnapshotContext) throws Exception {
  if (!isRunning) {
    LOG.debug("snapshotState() called on closed source");
  } else {

    if (checkpointCoder == null) {
      // no checkpoint coder available in this source
      return;
    }

    stateForCheckpoint.clear();

    long checkpointId = functionSnapshotContext.getCheckpointId();

    // we checkpoint the sources along with the CheckpointMarkT to ensure
    // than we have a correct mapping of checkpoints to sources when
    // restoring
    List<CheckpointMarkT> checkpointMarks = new ArrayList<>(localSplitSources.size());

    for (int i = 0; i < localSplitSources.size(); i++) {
      UnboundedSource<OutputT, CheckpointMarkT> source = localSplitSources.get(i);
      UnboundedSource.UnboundedReader<OutputT> reader = localReaders.get(i);

      @SuppressWarnings("unchecked")
      CheckpointMarkT mark = (CheckpointMarkT) reader.getCheckpointMark();
      checkpointMarks.add(mark);
      KV<UnboundedSource<OutputT, CheckpointMarkT>, CheckpointMarkT> kv = KV.of(source, mark);
      stateForCheckpoint.add(kv);
    }

    // cleanup old pending checkpoints and add new checkpoint
    int diff = pendingCheckpoints.size() - MAX_NUMBER_PENDING_CHECKPOINTS;
    if (diff >= 0) {
      for (Iterator<Long> iterator = pendingCheckpoints.keySet().iterator(); diff >= 0; diff--) {
        iterator.next();
        iterator.remove();
      }
    }
    pendingCheckpoints.put(checkpointId, checkpointMarks);
  }
}