Java Code Examples for org.apache.flink.runtime.checkpoint.CheckpointOptions#getCheckpointType()

/**
 * Triggers an asynchronous snapshot of the keyed state backend from RocksDB. This snapshot can be canceled and
 * is also stopped when the backend is closed through {@link #dispose()}. For each backend, this method must always
 * be called by the same thread.
 *
 * @param checkpointId  The Id of the checkpoint.
 * @param timestamp     The timestamp of the checkpoint.
 * @param streamFactory The factory that we can use for writing our state to streams.
 * @param checkpointOptions Options for how to perform this checkpoint.
 * @return Future to the state handle of the snapshot data.
 * @throws Exception indicating a problem in the synchronous part of the checkpoint.
 */
@Nonnull
@Override
public RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot(
	final long checkpointId,
	final long timestamp,
	@Nonnull final CheckpointStreamFactory streamFactory,
	@Nonnull CheckpointOptions checkpointOptions) throws Exception {

	long startTime = System.currentTimeMillis();

	// flush everything into db before taking a snapshot
	writeBatchWrapper.flush();

	RocksDBSnapshotStrategyBase<K> chosenSnapshotStrategy =
		CheckpointType.SAVEPOINT == checkpointOptions.getCheckpointType() ?
			savepointSnapshotStrategy : checkpointSnapshotStrategy;

	RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshotRunner =
		chosenSnapshotStrategy.snapshot(checkpointId, timestamp, streamFactory, checkpointOptions);

	chosenSnapshotStrategy.logSyncCompleted(streamFactory, startTime);

	return snapshotRunner;
}
 

private SupplierWithException<CheckpointStreamWithResultProvider, Exception> createCheckpointStreamSupplier(
	long checkpointId,
	CheckpointStreamFactory primaryStreamFactory,
	CheckpointOptions checkpointOptions) {

	return localRecoveryConfig.isLocalRecoveryEnabled() &&
		(CheckpointType.SAVEPOINT != checkpointOptions.getCheckpointType()) ?

		() -> CheckpointStreamWithResultProvider.createDuplicatingStream(
			checkpointId,
			CheckpointedStateScope.EXCLUSIVE,
			primaryStreamFactory,
			localRecoveryConfig.getLocalStateDirectoryProvider()) :

		() -> CheckpointStreamWithResultProvider.createSimpleStream(
			CheckpointedStateScope.EXCLUSIVE,
			primaryStreamFactory);
}
 

@Override
public boolean triggerCheckpoint(CheckpointMetaData checkpointMetaData, CheckpointOptions checkpointOptions, boolean advanceToEndOfEventTime) throws Exception {
	final long checkpointId = checkpointMetaData.getCheckpointId();
	final CheckpointType checkpointType = checkpointOptions.getCheckpointType();

	if (checkpointType == CheckpointType.SYNC_SAVEPOINT) {
		synchronousSavepointId = checkpointId;
		syncSavepointId.compareAndSet(-1, synchronousSavepointId);
	}

	final long taskIndex = getEnvironment().getTaskInfo().getIndexOfThisSubtask();
	if (taskIndex == 0) {
		checkpointsToWaitFor.countDown();
	}
	return super.triggerCheckpoint(checkpointMetaData, checkpointOptions, advanceToEndOfEventTime);
}
 

private void triggerCheckpointHelper(long checkpointId, long timestamp, CheckpointOptions checkpointOptions, boolean advanceToEndOfEventTime) {

		final CheckpointType checkpointType = checkpointOptions.getCheckpointType();
		if (advanceToEndOfEventTime && !(checkpointType.isSynchronous() && checkpointType.isSavepoint())) {
			throw new IllegalArgumentException("Only synchronous savepoints are allowed to advance the watermark to MAX.");
		}

		final LogicalSlot slot = assignedResource;

		if (slot != null) {
			final TaskManagerGateway taskManagerGateway = slot.getTaskManagerGateway();

			taskManagerGateway.triggerCheckpoint(attemptId, getVertex().getJobId(), checkpointId, timestamp, checkpointOptions, advanceToEndOfEventTime);
		} else {
			LOG.debug("The execution has no slot assigned. This indicates that the execution is no longer running.");
		}
	}
 

private void triggerCheckpointHelper(long checkpointId, long timestamp, CheckpointOptions checkpointOptions, boolean advanceToEndOfEventTime) {

		final CheckpointType checkpointType = checkpointOptions.getCheckpointType();
		if (advanceToEndOfEventTime && !(checkpointType.isSynchronous() && checkpointType.isSavepoint())) {
			throw new IllegalArgumentException("Only synchronous savepoints are allowed to advance the watermark to MAX.");
		}

		final LogicalSlot slot = assignedResource;

		if (slot != null) {
			final TaskManagerGateway taskManagerGateway = slot.getTaskManagerGateway();

			taskManagerGateway.triggerCheckpoint(attemptId, getVertex().getJobId(), checkpointId, timestamp, checkpointOptions, advanceToEndOfEventTime);
		} else {
			LOG.debug("The execution has no slot assigned. This indicates that the execution is no longer running.");
		}
	}
 

private static ByteBuffer serializeCheckpointBarrier(CheckpointBarrier barrier) throws IOException {
	final CheckpointOptions checkpointOptions = barrier.getCheckpointOptions();
	final CheckpointType checkpointType = checkpointOptions.getCheckpointType();

	final byte[] locationBytes = checkpointOptions.getTargetLocation().isDefaultReference() ?
			null : checkpointOptions.getTargetLocation().getReferenceBytes();

	final ByteBuffer buf = ByteBuffer.allocate(28 + (locationBytes == null ? 0 : locationBytes.length));

	// we do not use checkpointType.ordinal() here to make the serialization robust
	// against changes in the enum (such as changes in the order of the values)
	final int typeInt;
	if (checkpointType == CheckpointType.CHECKPOINT) {
		typeInt = CHECKPOINT_TYPE_CHECKPOINT;
	} else if (checkpointType == CheckpointType.SAVEPOINT) {
		typeInt = CHECKPOINT_TYPE_SAVEPOINT;
	} else {
		throw new IOException("Unknown checkpoint type: " + checkpointType);
	}

	buf.putInt(CHECKPOINT_BARRIER_EVENT);
	buf.putLong(barrier.getId());
	buf.putLong(barrier.getTimestamp());
	buf.putInt(typeInt);

	if (locationBytes == null) {
		buf.putInt(-1);
	} else {
		buf.putInt(locationBytes.length);
		buf.put(locationBytes);
	}

	buf.flip();
	return buf;
}
 

@Override
public CompletableFuture<Acknowledge> triggerCheckpoint(
		ExecutionAttemptID executionAttemptID,
		long checkpointId,
		long checkpointTimestamp,
		CheckpointOptions checkpointOptions,
		boolean advanceToEndOfEventTime) {
	log.debug("Trigger checkpoint {}@{} for {}.", checkpointId, checkpointTimestamp, executionAttemptID);

	final CheckpointType checkpointType = checkpointOptions.getCheckpointType();
	if (advanceToEndOfEventTime && !(checkpointType.isSynchronous() && checkpointType.isSavepoint())) {
		throw new IllegalArgumentException("Only synchronous savepoints are allowed to advance the watermark to MAX.");
	}

	final Task task = taskSlotTable.getTask(executionAttemptID);

	if (task != null) {
		task.triggerCheckpointBarrier(checkpointId, checkpointTimestamp, checkpointOptions, advanceToEndOfEventTime);

		return CompletableFuture.completedFuture(Acknowledge.get());
	} else {
		final String message = "TaskManager received a checkpoint request for unknown task " + executionAttemptID + '.';

		log.debug(message);
		return FutureUtils.completedExceptionally(new CheckpointException(message, CheckpointFailureReason.TASK_CHECKPOINT_FAILURE));
	}
}
 

private static ByteBuffer serializeCheckpointBarrier(CheckpointBarrier barrier) throws IOException {
	final CheckpointOptions checkpointOptions = barrier.getCheckpointOptions();
	final CheckpointType checkpointType = checkpointOptions.getCheckpointType();

	final byte[] locationBytes = checkpointOptions.getTargetLocation().isDefaultReference() ?
			null : checkpointOptions.getTargetLocation().getReferenceBytes();

	final ByteBuffer buf = ByteBuffer.allocate(28 + (locationBytes == null ? 0 : locationBytes.length));

	// we do not use checkpointType.ordinal() here to make the serialization robust
	// against changes in the enum (such as changes in the order of the values)
	final int typeInt;
	if (checkpointType == CheckpointType.CHECKPOINT) {
		typeInt = CHECKPOINT_TYPE_CHECKPOINT;
	} else if (checkpointType == CheckpointType.SAVEPOINT) {
		typeInt = CHECKPOINT_TYPE_SAVEPOINT;
	} else if (checkpointType == CheckpointType.SYNC_SAVEPOINT) {
		typeInt = CHECKPOINT_TYPE_SYNC_SAVEPOINT;
	} else {
		throw new IOException("Unknown checkpoint type: " + checkpointType);
	}

	buf.putInt(CHECKPOINT_BARRIER_EVENT);
	buf.putLong(barrier.getId());
	buf.putLong(barrier.getTimestamp());
	buf.putInt(typeInt);

	if (locationBytes == null) {
		buf.putInt(-1);
	} else {
		buf.putInt(locationBytes.length);
		buf.put(locationBytes);
	}

	buf.flip();
	return buf;
}
 

@Override
public Future<Boolean> triggerCheckpointAsync(CheckpointMetaData checkpointMetaData, CheckpointOptions checkpointOptions, boolean advanceToEndOfEventTime) {
	final long checkpointId = checkpointMetaData.getCheckpointId();
	final CheckpointType checkpointType = checkpointOptions.getCheckpointType();

	if (checkpointType == CheckpointType.SYNC_SAVEPOINT) {
		synchronousSavepointId = checkpointId;
		syncSavepointId.compareAndSet(-1, synchronousSavepointId);
	}

	return super.triggerCheckpointAsync(checkpointMetaData, checkpointOptions, advanceToEndOfEventTime);
}
 

@Override
public CompletableFuture<Acknowledge> triggerCheckpoint(
		ExecutionAttemptID executionAttemptID,
		long checkpointId,
		long checkpointTimestamp,
		CheckpointOptions checkpointOptions,
		boolean advanceToEndOfEventTime) {
	log.debug("Trigger checkpoint {}@{} for {}.", checkpointId, checkpointTimestamp, executionAttemptID);

	final CheckpointType checkpointType = checkpointOptions.getCheckpointType();
	if (advanceToEndOfEventTime && !(checkpointType.isSynchronous() && checkpointType.isSavepoint())) {
		throw new IllegalArgumentException("Only synchronous savepoints are allowed to advance the watermark to MAX.");
	}

	final Task task = taskSlotTable.getTask(executionAttemptID);

	if (task != null) {
		task.triggerCheckpointBarrier(checkpointId, checkpointTimestamp, checkpointOptions, advanceToEndOfEventTime);

		return CompletableFuture.completedFuture(Acknowledge.get());
	} else {
		final String message = "TaskManager received a checkpoint request for unknown task " + executionAttemptID + '.';

		log.debug(message);
		return FutureUtils.completedExceptionally(new CheckpointException(message, CheckpointFailureReason.TASK_CHECKPOINT_FAILURE));
	}
}
 

private static ByteBuffer serializeCheckpointBarrier(CheckpointBarrier barrier) throws IOException {
	final CheckpointOptions checkpointOptions = barrier.getCheckpointOptions();
	final CheckpointType checkpointType = checkpointOptions.getCheckpointType();

	final byte[] locationBytes = checkpointOptions.getTargetLocation().isDefaultReference() ?
			null : checkpointOptions.getTargetLocation().getReferenceBytes();

	final ByteBuffer buf = ByteBuffer.allocate(30 + (locationBytes == null ? 0 : locationBytes.length));

	// we do not use checkpointType.ordinal() here to make the serialization robust
	// against changes in the enum (such as changes in the order of the values)
	final int typeInt;
	if (checkpointType == CheckpointType.CHECKPOINT) {
		typeInt = CHECKPOINT_TYPE_CHECKPOINT;
	} else if (checkpointType == CheckpointType.SAVEPOINT) {
		typeInt = CHECKPOINT_TYPE_SAVEPOINT;
	} else if (checkpointType == CheckpointType.SYNC_SAVEPOINT) {
		typeInt = CHECKPOINT_TYPE_SYNC_SAVEPOINT;
	} else {
		throw new IOException("Unknown checkpoint type: " + checkpointType);
	}

	buf.putInt(CHECKPOINT_BARRIER_EVENT);
	buf.putLong(barrier.getId());
	buf.putLong(barrier.getTimestamp());
	buf.putInt(typeInt);

	if (locationBytes == null) {
		buf.putInt(-1);
	} else {
		buf.putInt(locationBytes.length);
		buf.put(locationBytes);
	}
	buf.put((byte) (checkpointOptions.isExactlyOnceMode() ? 1 : 0));
	buf.put((byte) (checkpointOptions.isUnalignedCheckpoint() ? 1 : 0));

	buf.flip();
	return buf;
}