Java Code Examples for org.apache.flink.util.ExceptionUtils#rethrowIfFatalErrorOrOOM()

/**
 * Loads the factories for the file systems directly supported by Flink.
 * Aside from the {@link LocalFileSystem}, these file systems are loaded
 * via Java's service framework.
 *
 * @return A map from the file system scheme to corresponding file system factory.
 */
private static List<FileSystemFactory> loadFileSystemFactories(
	Collection<Supplier<Iterator<FileSystemFactory>>> factoryIteratorsSuppliers) {

	final ArrayList<FileSystemFactory> list = new ArrayList<>();

	// by default, we always have the local file system factory
	list.add(new LocalFileSystemFactory());

	LOG.debug("Loading extension file systems via services");

	for (Supplier<Iterator<FileSystemFactory>> factoryIteratorsSupplier : factoryIteratorsSuppliers) {
		try {
			addAllFactoriesToList(factoryIteratorsSupplier.get(), list);
		} catch (Throwable t) {
			// catching Throwable here to handle various forms of class loading
			// and initialization errors
			ExceptionUtils.rethrowIfFatalErrorOrOOM(t);
			LOG.error("Failed to load additional file systems via services", t);
		}
	}

	return Collections.unmodifiableList(list);
}
 

/**
 * Updates the state of one of the ExecutionVertex's Execution attempts.
 * If the new status if "FINISHED", this also updates the accumulators.
 *
 * @param state The state update.
 * @return True, if the task update was properly applied, false, if the execution attempt was not found.
 */
public boolean updateState(TaskExecutionState state) {
	assertRunningInJobMasterMainThread();
	final Execution attempt = currentExecutions.get(state.getID());

	if (attempt != null) {
		try {
			final boolean stateUpdated = updateStateInternal(state, attempt);
			maybeReleasePartitions(attempt);
			return stateUpdated;
		}
		catch (Throwable t) {
			ExceptionUtils.rethrowIfFatalErrorOrOOM(t);

			// failures during updates leave the ExecutionGraph inconsistent
			failGlobal(t);
			return false;
		}
	}
	else {
		return false;
	}
}
 

private void checkpointCoordinatorInternal(final long checkpointId, final CompletableFuture<byte[]> result) {
	mainThreadExecutor.assertRunningInMainThread();

	// synchronously!!!, with the completion, we need to shut the event valve
	result.whenComplete((success, failure) -> {
		if (failure != null) {
			result.completeExceptionally(failure);
		} else {
			try {
				eventValve.shutValve(checkpointId);
				result.complete(success);
			} catch (Exception e) {
				result.completeExceptionally(e);
			}
		}
	});

	try {
		eventValve.markForCheckpoint(checkpointId);
		coordinator.checkpointCoordinator(checkpointId, result);
	} catch (Throwable t) {
		ExceptionUtils.rethrowIfFatalErrorOrOOM(t);
		result.completeExceptionally(t);
		globalFailureHandler.accept(t);
	}
}
 

/**
 * Handle asynchronous {@link Runnable}. This method simply executes the given {@link Runnable}
 * in the context of the actor thread.
 *
 * @param runAsync Run async message
 */
private void handleRunAsync(RunAsync runAsync) {
	final long timeToRun = runAsync.getTimeNanos();
	final long delayNanos;

	if (timeToRun == 0 || (delayNanos = timeToRun - System.nanoTime()) <= 0) {
		// run immediately
		try {
			runAsync.getRunnable().run();
		} catch (Throwable t) {
			log.error("Caught exception while executing runnable in main thread.", t);
			ExceptionUtils.rethrowIfFatalErrorOrOOM(t);
		}
	}
	else {
		// schedule for later. send a new message after the delay, which will then be immediately executed
		FiniteDuration delay = new FiniteDuration(delayNanos, TimeUnit.NANOSECONDS);
		RunAsync message = new RunAsync(runAsync.getRunnable(), timeToRun);

		final Object envelopedSelfMessage = envelopeSelfMessage(message);

		getContext().system().scheduler().scheduleOnce(delay, getSelf(), envelopedSelfMessage,
				getContext().dispatcher(), ActorRef.noSender());
	}
}
 

/**
 * Dispatches an operator event to the invokable task.
 *
 * <p>If the event delivery did not succeed, this method throws an exception. Callers can use that
 * exception for error reporting, but need not react with failing this task (this method takes care
 * of that).
 *
 * @throws FlinkException This method throws exceptions indicating the reason why delivery did not succeed.
 */
public void deliverOperatorEvent(OperatorID operator, SerializedValue<OperatorEvent> evt) throws FlinkException {
	final AbstractInvokable invokable = this.invokable;

	if (invokable == null || executionState != ExecutionState.RUNNING) {
		throw new TaskNotRunningException("Task is not yet running.");
	}

	try {
		invokable.dispatchOperatorEvent(operator, evt);
	}
	catch (Throwable t) {
		ExceptionUtils.rethrowIfFatalErrorOrOOM(t);

		if (getExecutionState() == ExecutionState.RUNNING) {
			FlinkException e = new FlinkException("Error while handling operator event", t);
			failExternally(e);
			throw e;
		}
	}
}
 

/**
 * Resets the master hooks.
 *
 * @param hooks The hooks to reset
 *
 * @throws FlinkException Thrown, if the hooks throw an exception.
 */
public static void reset(
	final Collection<MasterTriggerRestoreHook<?>> hooks,
	@SuppressWarnings("unused") final Logger log) throws FlinkException {

	for (MasterTriggerRestoreHook<?> hook : hooks) {
		final String id = hook.getIdentifier();
		try {
			hook.reset();
		}
		catch (Throwable t) {
			ExceptionUtils.rethrowIfFatalErrorOrOOM(t);
			throw new FlinkException("Error while resetting checkpoint master hook '" + id + '\'', t);
		}
	}
}
 

private static void addAllFactoriesToList(Iterator<FileSystemFactory> iter, List<FileSystemFactory> list) {
	// we explicitly use an iterator here (rather than for-each) because that way
	// we can catch errors in individual service instantiations

	while (iter.hasNext()) {
		try {
			FileSystemFactory factory = iter.next();
			list.add(factory);
			LOG.debug("Added file system {}:{}", factory.getScheme(), factory.getClass().getName());
		}
		catch (Throwable t) {
			// catching Throwable here to handle various forms of class loading
			// and initialization errors
			ExceptionUtils.rethrowIfFatalErrorOrOOM(t);
			LOG.error("Failed to load a file system via services", t);
		}
	}
}
 

/**
 * Resets the master hooks.
 *
 * @param hooks The hooks to reset
 *
 * @throws FlinkException Thrown, if the hooks throw an exception.
 */
public static void reset(
	final Collection<MasterTriggerRestoreHook<?>> hooks,
	final Logger log) throws FlinkException {

	for (MasterTriggerRestoreHook<?> hook : hooks) {
		final String id = hook.getIdentifier();
		try {
			hook.reset();
		}
		catch (Throwable t) {
			ExceptionUtils.rethrowIfFatalErrorOrOOM(t);
			throw new FlinkException("Error while resetting checkpoint master hook '" + id + '\'', t);
		}
	}
}
 

/**
 * Updates the state of one of the ExecutionVertex's Execution attempts.
 * If the new status if "FINISHED", this also updates the accumulators.
 *
 * @param state The state update.
 * @return True, if the task update was properly applied, false, if the execution attempt was not found.
 */
public boolean updateState(TaskExecutionState state) {
	assertRunningInJobMasterMainThread();
	final Execution attempt = currentExecutions.get(state.getID());

	if (attempt != null) {
		try {
			final boolean stateUpdated = updateStateInternal(state, attempt);
			maybeReleasePartitions(attempt);
			return stateUpdated;
		}
		catch (Throwable t) {
			ExceptionUtils.rethrowIfFatalErrorOrOOM(t);

			// failures during updates leave the ExecutionGraph inconsistent
			failGlobal(t);
			return false;
		}
	}
	else {
		return false;
	}
}
 

/**
 * Handle asynchronous {@link Runnable}. This method simply executes the given {@link Runnable}
 * in the context of the actor thread.
 *
 * @param runAsync Run async message
 */
private void handleRunAsync(RunAsync runAsync) {
	final long timeToRun = runAsync.getTimeNanos();
	final long delayNanos;

	if (timeToRun == 0 || (delayNanos = timeToRun - System.nanoTime()) <= 0) {
		// run immediately
		try {
			runAsync.getRunnable().run();
		} catch (Throwable t) {
			log.error("Caught exception while executing runnable in main thread.", t);
			ExceptionUtils.rethrowIfFatalErrorOrOOM(t);
		}
	}
	else {
		// schedule for later. send a new message after the delay, which will then be immediately executed
		FiniteDuration delay = new FiniteDuration(delayNanos, TimeUnit.NANOSECONDS);
		RunAsync message = new RunAsync(runAsync.getRunnable(), timeToRun);

		final Object envelopedSelfMessage = envelopeSelfMessage(message);

		getContext().system().scheduler().scheduleOnce(delay, getSelf(), envelopedSelfMessage,
				getContext().dispatcher(), ActorRef.noSender());
	}
}
 

/**
 * Resets the master hooks.
 *
 * @param hooks The hooks to reset
 *
 * @throws FlinkException Thrown, if the hooks throw an exception.
 */
public static void reset(
	final Collection<MasterTriggerRestoreHook<?>> hooks,
	final Logger log) throws FlinkException {

	for (MasterTriggerRestoreHook<?> hook : hooks) {
		final String id = hook.getIdentifier();
		try {
			hook.reset();
		}
		catch (Throwable t) {
			ExceptionUtils.rethrowIfFatalErrorOrOOM(t);
			throw new FlinkException("Error while resetting checkpoint master hook '" + id + '\'', t);
		}
	}
}
 

private static void addAllFactoriesToList(Iterator<FileSystemFactory> iter, List<FileSystemFactory> list) {
	// we explicitly use an iterator here (rather than for-each) because that way
	// we can catch errors in individual service instantiations

	while (iter.hasNext()) {
		try {
			FileSystemFactory factory = iter.next();
			list.add(factory);
			LOG.debug("Added file system {}:{}", factory.getScheme(), factory.getClass().getName());
		}
		catch (Throwable t) {
			// catching Throwable here to handle various forms of class loading
			// and initialization errors
			ExceptionUtils.rethrowIfFatalErrorOrOOM(t);
			LOG.error("Failed to load a file system via services", t);
		}
	}
}
 

/**
 * Loads the factories for the file systems directly supported by Flink.
 * Aside from the {@link LocalFileSystem}, these file systems are loaded
 * via Java's service framework.
 *
 * @return A map from the file system scheme to corresponding file system factory.
 */
private static List<FileSystemFactory> loadFileSystemFactories(
	Collection<Supplier<Iterator<FileSystemFactory>>> factoryIteratorsSuppliers) {

	final ArrayList<FileSystemFactory> list = new ArrayList<>();

	// by default, we always have the local file system factory
	list.add(new LocalFileSystemFactory());

	LOG.debug("Loading extension file systems via services");

	for (Supplier<Iterator<FileSystemFactory>> factoryIteratorsSupplier : factoryIteratorsSuppliers) {
		try {
			addAllFactoriesToList(factoryIteratorsSupplier.get(), list);
		} catch (Throwable t) {
			// catching Throwable here to handle various forms of class loading
			// and initialization errors
			ExceptionUtils.rethrowIfFatalErrorOrOOM(t);
			LOG.error("Failed to load additional file systems via services", t);
		}
	}

	return Collections.unmodifiableList(list);
}
 

/**
 * Handle asynchronous {@link Runnable}. This method simply executes the given {@link Runnable}
 * in the context of the actor thread.
 *
 * @param runAsync Run async message
 */
private void handleRunAsync(RunAsync runAsync) {
	final long timeToRun = runAsync.getTimeNanos();
	final long delayNanos;

	if (timeToRun == 0 || (delayNanos = timeToRun - System.nanoTime()) <= 0) {
		// run immediately
		try {
			runAsync.getRunnable().run();
		} catch (Throwable t) {
			log.error("Caught exception while executing runnable in main thread.", t);
			ExceptionUtils.rethrowIfFatalErrorOrOOM(t);
		}
	}
	else {
		// schedule for later. send a new message after the delay, which will then be immediately executed
		FiniteDuration delay = new FiniteDuration(delayNanos, TimeUnit.NANOSECONDS);
		RunAsync message = new RunAsync(runAsync.getRunnable(), timeToRun);

		final Object envelopedSelfMessage = envelopeSelfMessage(message);

		getContext().system().scheduler().scheduleOnce(delay, getSelf(), envelopedSelfMessage,
				getContext().dispatcher(), ActorRef.noSender());
	}
}
 

@Override
public void deliverOperatorEventToCoordinator(
		final ExecutionAttemptID taskExecutionId,
		final OperatorID operatorId,
		final OperatorEvent evt) throws FlinkException {

	// Failure semantics (as per the javadocs of the method):
	// If the task manager sends an event for a non-running task or an non-existing operator
	// coordinator, then respond with an exception to the call. If task and coordinator exist,
	// then we assume that the call from the TaskManager was valid, and any bubbling exception
	// needs to cause a job failure.

	final Execution exec = executionGraph.getRegisteredExecutions().get(taskExecutionId);
	if (exec == null || exec.getState() != ExecutionState.RUNNING) {
		// This situation is common when cancellation happens, or when the task failed while the
		// event was just being dispatched asynchronously on the TM side.
		// It should be fine in those expected situations to just ignore this event, but, to be
		// on the safe, we notify the TM that the event could not be delivered.
		throw new TaskNotRunningException("Task is not known or in state running on the JobManager.");
	}

	final OperatorCoordinatorHolder coordinator = coordinatorMap.get(operatorId);
	if (coordinator == null) {
		throw new FlinkException("No coordinator registered for operator " + operatorId);
	}

	try {
		coordinator.handleEventFromOperator(exec.getParallelSubtaskIndex(), evt);
	} catch (Throwable t) {
		ExceptionUtils.rethrowIfFatalErrorOrOOM(t);
		handleGlobalFailure(t);
	}
}
 

private void startAllOperatorCoordinators() {
	final Collection<OperatorCoordinatorHolder> coordinators = getAllCoordinators();
	try {
		for (OperatorCoordinatorHolder coordinator : coordinators) {
			coordinator.start();
		}
	}
	catch (Throwable t) {
		ExceptionUtils.rethrowIfFatalErrorOrOOM(t);
		coordinators.forEach(IOUtils::closeQuietly);
		throw new FlinkRuntimeException("Failed to start the operator coordinators", t);
	}
}
 

/**
 * Updates the state of one of the ExecutionVertex's Execution attempts.
 * If the new status if "FINISHED", this also updates the accumulators.
 *
 * @param state The state update.
 * @return True, if the task update was properly applied, false, if the execution attempt was not found.
 */
public boolean updateState(TaskExecutionState state) {
	assertRunningInJobMasterMainThread();
	final Execution attempt = currentExecutions.get(state.getID());

	if (attempt != null) {
		try {
			Map<String, Accumulator<?, ?>> accumulators;

			switch (state.getExecutionState()) {
				case RUNNING:
					return attempt.switchToRunning();

				case FINISHED:
					// this deserialization is exception-free
					accumulators = deserializeAccumulators(state);
					attempt.markFinished(accumulators, state.getIOMetrics());
					return true;

				case CANCELED:
					// this deserialization is exception-free
					accumulators = deserializeAccumulators(state);
					attempt.completeCancelling(accumulators, state.getIOMetrics());
					return true;

				case FAILED:
					// this deserialization is exception-free
					accumulators = deserializeAccumulators(state);
					attempt.markFailed(state.getError(userClassLoader), accumulators, state.getIOMetrics());
					return true;

				default:
					// we mark as failed and return false, which triggers the TaskManager
					// to remove the task
					attempt.fail(new Exception("TaskManager sent illegal state update: " + state.getExecutionState()));
					return false;
			}
		}
		catch (Throwable t) {
			ExceptionUtils.rethrowIfFatalErrorOrOOM(t);

			// failures during updates leave the ExecutionGraph inconsistent
			failGlobal(t);
			return false;
		}
	}
	else {
		return false;
	}
}