Java Code Examples for org.apache.flink.runtime.checkpoint.CheckpointMetaData#getCheckpointId()

@Override
public boolean triggerCheckpoint(CheckpointMetaData checkpointMetaData, CheckpointOptions checkpointOptions) {
	lastCheckpointId++;
	if (checkpointMetaData.getCheckpointId() == lastCheckpointId) {
		if (lastCheckpointId == numCalls) {
			triggerLatch.trigger();
		}
	}
	else if (this.error == null) {
		this.error = new Exception("calls out of order");
		synchronized (this) {
			notifyAll();
		}
	}
	return true;
}
 

public void triggerCheckpointOnBarrier(
		CheckpointMetaData checkpointMetaData,
		CheckpointOptions checkpointOptions,
		CheckpointMetrics checkpointMetrics) throws IOException {
	if (nextExpectedCheckpointId != -1L) {
		assertEquals("wrong checkpoint id", nextExpectedCheckpointId, checkpointMetaData.getCheckpointId());
	}

	assertTrue(checkpointMetaData.getTimestamp() > 0);
	assertTrue(checkpointMetrics.getAlignmentDurationNanos() >= 0);

	nextExpectedCheckpointId = checkpointMetaData.getCheckpointId() + 1;

	for (int index = 0; index < inputGate.getNumberOfInputChannels(); index++) {
		inputGate.getChannel(index).spillInflightBuffers(checkpointMetaData.getCheckpointId(), NO_OP);
	}
}
 

@Override
public Future<Boolean> triggerCheckpointAsync(CheckpointMetaData checkpointMetaData, CheckpointOptions checkpointOptions, boolean advanceToEndOfEventTime) {
	lastCheckpointId++;
	if (checkpointMetaData.getCheckpointId() == lastCheckpointId) {
		if (lastCheckpointId == numCalls) {
			triggerLatch.trigger();
		}
	}
	else if (this.error == null) {
		this.error = new Exception("calls out of order");
		synchronized (this) {
			notifyAll();
		}
	}
	return CompletableFuture.completedFuture(true);
}
 

@Override
public void reportTaskStateSnapshots(
	@Nonnull CheckpointMetaData checkpointMetaData,
	@Nonnull CheckpointMetrics checkpointMetrics,
	@Nullable TaskStateSnapshot acknowledgedState,
	@Nullable TaskStateSnapshot localState) {

	long checkpointId = checkpointMetaData.getCheckpointId();

	localStateStore.storeLocalState(checkpointId, localState);

	checkpointResponder.acknowledgeCheckpoint(
		jobId,
		executionAttemptID,
		checkpointId,
		checkpointMetrics,
		acknowledgedState);
}
 

@Override
public boolean triggerCheckpoint(
		CheckpointMetaData checkpointMetaData,
		CheckpointOptions checkpointOptions,
		boolean advanceToEndOfEventTime) throws Exception {

	try {
		// No alignment if we inject a checkpoint
		CheckpointMetrics checkpointMetrics = new CheckpointMetrics()
				.setBytesBufferedInAlignment(0L)
				.setAlignmentDurationNanos(0L);

		return performCheckpoint(checkpointMetaData, checkpointOptions, checkpointMetrics, advanceToEndOfEventTime);
	}
	catch (Exception e) {
		// propagate exceptions only if the task is still in "running" state
		if (isRunning) {
			throw new Exception("Could not perform checkpoint " + checkpointMetaData.getCheckpointId() +
				" for operator " + getName() + '.', e);
		} else {
			LOG.debug("Could not perform checkpoint {} for operator {} while the " +
				"invokable was not in state running.", checkpointMetaData.getCheckpointId(), getName(), e);
			return false;
		}
	}
}
 

@Override
public boolean triggerCheckpoint(CheckpointMetaData checkpointMetaData, CheckpointOptions checkpointOptions, boolean advanceToEndOfEventTime) {
	lastCheckpointId++;
	if (checkpointMetaData.getCheckpointId() == lastCheckpointId) {
		if (lastCheckpointId == numCalls) {
			triggerLatch.trigger();
		}
	}
	else if (this.error == null) {
		this.error = new Exception("calls out of order");
		synchronized (this) {
			notifyAll();
		}
	}
	return true;
}
 

@Override
public void reportTaskStateSnapshots(
	@Nonnull CheckpointMetaData checkpointMetaData,
	@Nonnull CheckpointMetrics checkpointMetrics,
	@Nullable TaskStateSnapshot acknowledgedState,
	@Nullable TaskStateSnapshot localState) {

	long checkpointId = checkpointMetaData.getCheckpointId();

	localStateStore.storeLocalState(checkpointId, localState);

	checkpointResponder.acknowledgeCheckpoint(
		jobId,
		executionAttemptID,
		checkpointId,
		checkpointMetrics,
		acknowledgedState);
}
 

@Override
public boolean triggerCheckpoint(CheckpointMetaData checkpointMetaData, CheckpointOptions checkpointOptions, boolean advanceToEndOfEventTime) throws Exception {
	final long checkpointId = checkpointMetaData.getCheckpointId();
	final CheckpointType checkpointType = checkpointOptions.getCheckpointType();

	if (checkpointType == CheckpointType.SYNC_SAVEPOINT) {
		synchronousSavepointId = checkpointId;
		syncSavepointId.compareAndSet(-1, synchronousSavepointId);
	}

	final long taskIndex = getEnvironment().getTaskInfo().getIndexOfThisSubtask();
	if (taskIndex == 0) {
		checkpointsToWaitFor.countDown();
	}
	return super.triggerCheckpoint(checkpointMetaData, checkpointOptions, advanceToEndOfEventTime);
}
 

@Override
public boolean triggerCheckpoint(CheckpointMetaData checkpointMetaData, CheckpointOptions checkpointOptions) throws Exception {
	try {
		// No alignment if we inject a checkpoint
		CheckpointMetrics checkpointMetrics = new CheckpointMetrics()
				.setBytesBufferedInAlignment(0L)
				.setAlignmentDurationNanos(0L);

		return performCheckpoint(checkpointMetaData, checkpointOptions, checkpointMetrics);
	}
	catch (Exception e) {
		// propagate exceptions only if the task is still in "running" state
		if (isRunning) {
			throw new Exception("Could not perform checkpoint " + checkpointMetaData.getCheckpointId() +
				" for operator " + getName() + '.', e);
		} else {
			LOG.debug("Could not perform checkpoint {} for operator {} while the " +
				"invokable was not in state running.", checkpointMetaData.getCheckpointId(), getName(), e);
			return false;
		}
	}
}
 

@Override
public void reportTaskStateSnapshots(
	@Nonnull CheckpointMetaData checkpointMetaData,
	@Nonnull CheckpointMetrics checkpointMetrics,
	@Nullable TaskStateSnapshot acknowledgedState,
	@Nullable TaskStateSnapshot localState) {

	long checkpointId = checkpointMetaData.getCheckpointId();

	localStateStore.storeLocalState(checkpointId, localState);

	checkpointResponder.acknowledgeCheckpoint(
		jobId,
		executionAttemptID,
		checkpointId,
		checkpointMetrics,
		acknowledgedState);
}
 

@Override
public void reportTaskStateSnapshots(
	@Nonnull CheckpointMetaData checkpointMetaData,
	@Nonnull CheckpointMetrics checkpointMetrics,
	@Nullable TaskStateSnapshot acknowledgedState,
	@Nullable TaskStateSnapshot localState) {


	jobManagerTaskStateSnapshotsByCheckpointId.put(
		checkpointMetaData.getCheckpointId(),
		acknowledgedState);

	taskManagerTaskStateSnapshotsByCheckpointId.put(
		checkpointMetaData.getCheckpointId(),
		localState);

	if (checkpointResponder != null) {
		checkpointResponder.acknowledgeCheckpoint(
			jobId,
			executionAttemptID,
			checkpointMetaData.getCheckpointId(),
			checkpointMetrics,
			acknowledgedState);
	}

	this.reportedCheckpointId = checkpointMetaData.getCheckpointId();

	if (waitForReportLatch != null) {
		waitForReportLatch.trigger();
	}
}
 

@Override
public Future<Boolean> triggerCheckpointAsync(CheckpointMetaData checkpointMetaData, CheckpointOptions checkpointOptions, boolean advanceToEndOfEventTime) {
	final long checkpointId = checkpointMetaData.getCheckpointId();
	final CheckpointType checkpointType = checkpointOptions.getCheckpointType();

	if (checkpointType == CheckpointType.SYNC_SAVEPOINT) {
		synchronousSavepointId = checkpointId;
		syncSavepointId.compareAndSet(-1, synchronousSavepointId);
	}

	return super.triggerCheckpointAsync(checkpointMetaData, checkpointOptions, advanceToEndOfEventTime);
}
 

@Override
public void reportTaskStateSnapshots(
	@Nonnull CheckpointMetaData checkpointMetaData,
	@Nonnull CheckpointMetrics checkpointMetrics,
	@Nullable TaskStateSnapshot acknowledgedState,
	@Nullable TaskStateSnapshot localState) {


	jobManagerTaskStateSnapshotsByCheckpointId.put(
		checkpointMetaData.getCheckpointId(),
		acknowledgedState);

	taskManagerTaskStateSnapshotsByCheckpointId.put(
		checkpointMetaData.getCheckpointId(),
		localState);

	if (checkpointResponder != null) {
		checkpointResponder.acknowledgeCheckpoint(
			jobId,
			executionAttemptID,
			checkpointMetaData.getCheckpointId(),
			checkpointMetrics,
			acknowledgedState);
	}

	this.reportedCheckpointId = checkpointMetaData.getCheckpointId();

	if (waitForReportLatch != null) {
		waitForReportLatch.trigger();
	}
}
 

@Override
public void reportTaskStateSnapshots(
	@Nonnull CheckpointMetaData checkpointMetaData,
	@Nonnull CheckpointMetrics checkpointMetrics,
	@Nullable TaskStateSnapshot acknowledgedState,
	@Nullable TaskStateSnapshot localState) {

	jobManagerTaskStateSnapshotsByCheckpointId.put(
		checkpointMetaData.getCheckpointId(),
		acknowledgedState);

	taskManagerTaskStateSnapshotsByCheckpointId.put(
		checkpointMetaData.getCheckpointId(),
		localState);

	if (checkpointResponder != null) {
		checkpointResponder.acknowledgeCheckpoint(
			jobId,
			executionAttemptID,
			checkpointMetaData.getCheckpointId(),
			checkpointMetrics,
			acknowledgedState);
	}

	this.reportedCheckpointId = checkpointMetaData.getCheckpointId();

	if (waitForReportLatch != null) {
		waitForReportLatch.trigger();
	}
}
 

private boolean triggerCheckpoint(
		CheckpointMetaData checkpointMetaData,
		CheckpointOptions checkpointOptions,
		boolean advanceToEndOfEventTime) throws Exception {
	try {
		// No alignment if we inject a checkpoint
		CheckpointMetrics checkpointMetrics = new CheckpointMetrics().setAlignmentDurationNanos(0L);

		subtaskCheckpointCoordinator.initCheckpoint(checkpointMetaData.getCheckpointId(), checkpointOptions);

		boolean success = performCheckpoint(checkpointMetaData, checkpointOptions, checkpointMetrics, advanceToEndOfEventTime);
		if (!success) {
			declineCheckpoint(checkpointMetaData.getCheckpointId());
		}
		return success;
	} catch (Exception e) {
		// propagate exceptions only if the task is still in "running" state
		if (isRunning) {
			throw new Exception("Could not perform checkpoint " + checkpointMetaData.getCheckpointId() +
				" for operator " + getName() + '.', e);
		} else {
			LOG.debug("Could not perform checkpoint {} for operator {} while the " +
				"invokable was not in state running.", checkpointMetaData.getCheckpointId(), getName(), e);
			return false;
		}
	}
}
 

public void triggerCheckpointOnBarrier(
		CheckpointMetaData checkpointMetaData,
		CheckpointOptions checkpointOptions,
		CheckpointMetrics checkpointMetrics) {
	expectedCheckpointId = checkpointMetaData.getCheckpointId();
	totalNumCheckpoints++;
}
 

private boolean performCheckpoint(
		CheckpointMetaData checkpointMetaData,
		CheckpointOptions checkpointOptions,
		CheckpointMetrics checkpointMetrics) throws Exception {

	LOG.debug("Starting checkpoint ({}) {} on task {}",
		checkpointMetaData.getCheckpointId(), checkpointOptions.getCheckpointType(), getName());

	synchronized (lock) {
		if (isRunning) {
			// we can do a checkpoint

			// All of the following steps happen as an atomic step from the perspective of barriers and
			// records/watermarks/timers/callbacks.
			// We generally try to emit the checkpoint barrier as soon as possible to not affect downstream
			// checkpoint alignments

			// Step (1): Prepare the checkpoint, allow operators to do some pre-barrier work.
			//           The pre-barrier work should be nothing or minimal in the common case.
			operatorChain.prepareSnapshotPreBarrier(checkpointMetaData.getCheckpointId());

			// Step (2): Send the checkpoint barrier downstream
			operatorChain.broadcastCheckpointBarrier(
					checkpointMetaData.getCheckpointId(),
					checkpointMetaData.getTimestamp(),
					checkpointOptions);

			// Step (3): Take the state snapshot. This should be largely asynchronous, to not
			//           impact progress of the streaming topology
			checkpointState(checkpointMetaData, checkpointOptions, checkpointMetrics);
			return true;
		}
		else {
			// we cannot perform our checkpoint - let the downstream operators know that they
			// should not wait for any input from this operator

			// we cannot broadcast the cancellation markers on the 'operator chain', because it may not
			// yet be created
			final CancelCheckpointMarker message = new CancelCheckpointMarker(checkpointMetaData.getCheckpointId());
			Exception exception = null;

			for (RecordWriter<SerializationDelegate<StreamRecord<OUT>>> recordWriter : recordWriters) {
				try {
					recordWriter.broadcastEvent(message);
				} catch (Exception e) {
					exception = ExceptionUtils.firstOrSuppressed(
						new Exception("Could not send cancel checkpoint marker to downstream tasks.", e),
						exception);
				}
			}

			if (exception != null) {
				throw exception;
			}

			return false;
		}
	}
}
 

private boolean performCheckpoint(
		CheckpointMetaData checkpointMetaData,
		CheckpointOptions checkpointOptions,
		CheckpointMetrics checkpointMetrics,
		boolean advanceToEndOfTime) throws Exception {

	LOG.debug("Starting checkpoint ({}) {} on task {}",
		checkpointMetaData.getCheckpointId(), checkpointOptions.getCheckpointType(), getName());

	final long checkpointId = checkpointMetaData.getCheckpointId();

	synchronized (lock) {
		if (isRunning) {

			if (checkpointOptions.getCheckpointType().isSynchronous()) {
				syncSavepointLatch.setCheckpointId(checkpointId);

				if (advanceToEndOfTime) {
					advanceToEndOfEventTime();
				}
			}

			// All of the following steps happen as an atomic step from the perspective of barriers and
			// records/watermarks/timers/callbacks.
			// We generally try to emit the checkpoint barrier as soon as possible to not affect downstream
			// checkpoint alignments

			// Step (1): Prepare the checkpoint, allow operators to do some pre-barrier work.
			//           The pre-barrier work should be nothing or minimal in the common case.
			operatorChain.prepareSnapshotPreBarrier(checkpointId);

			// Step (2): Send the checkpoint barrier downstream
			operatorChain.broadcastCheckpointBarrier(
					checkpointId,
					checkpointMetaData.getTimestamp(),
					checkpointOptions);

			// Step (3): Take the state snapshot. This should be largely asynchronous, to not
			//           impact progress of the streaming topology
			checkpointState(checkpointMetaData, checkpointOptions, checkpointMetrics);

			return true;
		}
		else {
			// we cannot perform our checkpoint - let the downstream operators know that they
			// should not wait for any input from this operator

			// we cannot broadcast the cancellation markers on the 'operator chain', because it may not
			// yet be created
			final CancelCheckpointMarker message = new CancelCheckpointMarker(checkpointMetaData.getCheckpointId());
			Exception exception = null;

			for (RecordWriter<SerializationDelegate<StreamRecord<OUT>>> recordWriter : recordWriters) {
				try {
					recordWriter.broadcastEvent(message);
				} catch (Exception e) {
					exception = ExceptionUtils.firstOrSuppressed(
						new Exception("Could not send cancel checkpoint marker to downstream tasks.", e),
						exception);
				}
			}

			if (exception != null) {
				throw exception;
			}

			return false;
		}
	}
}
 

@Override
public void checkpointState(
		CheckpointMetaData metadata,
		CheckpointOptions options,
		CheckpointMetrics metrics,
		OperatorChain<?, ?> operatorChain,
		Supplier<Boolean> isCanceled) throws Exception {

	checkNotNull(options);
	checkNotNull(metrics);

	// All of the following steps happen as an atomic step from the perspective of barriers and
	// records/watermarks/timers/callbacks.
	// We generally try to emit the checkpoint barrier as soon as possible to not affect downstream
	// checkpoint alignments

	if (lastCheckpointId >= metadata.getCheckpointId()) {
		LOG.info("Out of order checkpoint barrier (aborted previously?): {} >= {}", lastCheckpointId, metadata.getCheckpointId());
		channelStateWriter.abort(metadata.getCheckpointId(), new CancellationException(), true);
		checkAndClearAbortedStatus(metadata.getCheckpointId());
		return;
	}

	// Step (0): Record the last triggered checkpointId and abort the sync phase of checkpoint if necessary.
	lastCheckpointId = metadata.getCheckpointId();
	if (checkAndClearAbortedStatus(metadata.getCheckpointId())) {
		// broadcast cancel checkpoint marker to avoid downstream back-pressure due to checkpoint barrier align.
		operatorChain.broadcastEvent(new CancelCheckpointMarker(metadata.getCheckpointId()));
		LOG.info("Checkpoint {} has been notified as aborted, would not trigger any checkpoint.", metadata.getCheckpointId());
		return;
	}

	// Step (1): Prepare the checkpoint, allow operators to do some pre-barrier work.
	//           The pre-barrier work should be nothing or minimal in the common case.
	operatorChain.prepareSnapshotPreBarrier(metadata.getCheckpointId());

	// Step (2): Send the checkpoint barrier downstream
	operatorChain.broadcastEvent(
		new CheckpointBarrier(metadata.getCheckpointId(), metadata.getTimestamp(), options),
		options.isUnalignedCheckpoint());

	// Step (3): Prepare to spill the in-flight buffers for input and output
	if (options.isUnalignedCheckpoint()) {
		prepareInflightDataSnapshot(metadata.getCheckpointId());
	}

	// Step (4): Take the state snapshot. This should be largely asynchronous, to not impact progress of the
	// streaming topology

	Map<OperatorID, OperatorSnapshotFutures> snapshotFutures = new HashMap<>(operatorChain.getNumberOfOperators());
	try {
		if (takeSnapshotSync(snapshotFutures, metadata, metrics, options, operatorChain, isCanceled)) {
			finishAndReportAsync(snapshotFutures, metadata, metrics, options);
		} else {
			cleanup(snapshotFutures, metadata, metrics, new Exception("Checkpoint declined"));
		}
	} catch (Exception ex) {
		cleanup(snapshotFutures, metadata, metrics, ex);
		throw ex;
	}
}