Java Code Examples for org.apache.flink.util.ExceptionUtils#firstOrSuppressed()

/**
 * Cancels all instances of {@link java.util.concurrent.Future} in the given list of runnables without interrupting.
 * This method will suppress unexpected exceptions until the whole list is processed and then rethrow.
 *
 * @param runnables list of {@link Runnable} candidates to cancel.
 */
public static void cancelRunnableFutures(@Nonnull List<Runnable> runnables) {
	RuntimeException suppressedExceptions = null;
	for (Runnable runnable : runnables) {
		if (runnable instanceof java.util.concurrent.Future) {
			try {
				((java.util.concurrent.Future<?>) runnable).cancel(false);
			} catch (RuntimeException ex) {
				// safety net to ensure all candidates get cancelled before we let the exception bubble up.
				suppressedExceptions = ExceptionUtils.firstOrSuppressed(ex, suppressedExceptions);
			}
		}
	}
	if (suppressedExceptions != null) {
		throw suppressedExceptions;
	}
}
 

@Override
public void broadcastEvent(AbstractEvent event) throws IOException {
	IOException exception = null;
	for (RecordWriter recordWriter : recordWriters) {
		try {
			recordWriter.broadcastEvent(event);
		} catch (IOException e) {
			exception = ExceptionUtils.firstOrSuppressed(
				new IOException("Could not send event to downstream tasks.", e), exception);
		}
	}

	if (exception != null) {
		throw exception;
	}
}
 

private T attemptCreateAndRestore(Collection<S> restoreState) throws Exception {

		// create a new backend with necessary initialization.
		final T backendInstance = instanceSupplier.apply(restoreState);

		try {
			// register the backend with the registry to participate in task lifecycle w.r.t. cancellation.
			backendCloseableRegistry.registerCloseable(backendInstance);
			return backendInstance;
		} catch (Exception ex) {
			// dispose the backend, e.g. to release native resources, if failed to register it into registry.
			try {
				backendInstance.dispose();
			} catch (Exception disposeEx) {
				ex = ExceptionUtils.firstOrSuppressed(disposeEx, ex);
			}

			throw ex;
		}
	}
 

@Override
public void discardState() throws Exception {

	Exception aggregatedExceptions = null;

	if (jobManagerOwnedSnapshot != null) {
		try {
			jobManagerOwnedSnapshot.discardState();
		} catch (Exception remoteDiscardEx) {
			aggregatedExceptions = remoteDiscardEx;
		}
	}

	if (taskLocalSnapshot != null) {
		try {
			taskLocalSnapshot.discardState();
		} catch (Exception localDiscardEx) {
			aggregatedExceptions = ExceptionUtils.firstOrSuppressed(localDiscardEx, aggregatedExceptions);
		}
	}

	if (aggregatedExceptions != null) {
		throw aggregatedExceptions;
	}
}
 

private void completeFuture(Object ignored, Throwable throwable) {
	synchronized (lock) {
		futuresCompleted++;

		if (throwable != null) {
			globalThrowable = ExceptionUtils.firstOrSuppressed(throwable, globalThrowable);
		}

		if (futuresCompleted == numFuturesTotal) {
			if (globalThrowable != null) {
				completeExceptionally(globalThrowable);
			} else {
				complete(null);
			}
		}
	}
}
 

/**
 * Execute @link StreamOperator#dispose()} of each operator in the chain of this
 * {@link StreamTask}. Disposing happens from <b>tail to head</b> operator in the chain.
 */
private void disposeAllOperators() throws Exception {
	if (operatorChain != null && !disposedOperators) {
		Exception disposalException = null;
		for (StreamOperatorWrapper<?, ?> operatorWrapper : operatorChain.getAllOperators(true)) {
			StreamOperator<?> operator = operatorWrapper.getStreamOperator();
			try {
				operator.dispose();
			}
			catch (Exception e) {
				disposalException = ExceptionUtils.firstOrSuppressed(e, disposalException);
			}
		}
		disposedOperators = true;
		if (disposalException != null) {
			throw disposalException;
		}
	}
}
 

/**
 * Stop and clear the currently registered job leader id listeners.
 *
 * @throws Exception which is thrown in case a retrieval service cannot be stopped properly
 */
public void clear() throws Exception {
	Exception exception = null;

	for (JobLeaderIdListener listener: jobLeaderIdListeners.values()) {
		try {
			listener.stop();
		} catch (Exception e) {
			exception = ExceptionUtils.firstOrSuppressed(e, exception);
		}
	}

	if (exception != null) {
		ExceptionUtils.rethrowException(exception, "Could not properly stop the " +
			JobLeaderIdService.class.getSimpleName() + '.');
	}

	jobLeaderIdListeners.clear();
}
 

private T attemptCreateAndRestore(Collection<S> restoreState) throws Exception {

		// create a new backend with necessary initialization.
		final T backendInstance = instanceSupplier.apply(restoreState);

		try {
			// register the backend with the registry to participate in task lifecycle w.r.t. cancellation.
			backendCloseableRegistry.registerCloseable(backendInstance);
			return backendInstance;
		} catch (Exception ex) {
			// dispose the backend, e.g. to release native resources, if failed to register it into registry.
			try {
				backendInstance.dispose();
			} catch (Exception disposeEx) {
				ex = ExceptionUtils.firstOrSuppressed(disposeEx, ex);
			}

			throw ex;
		}
	}
 

/**
 * Releases all lock nodes of this ZooKeeperStateHandleStore.
 *
 * @throws Exception if the delete operation of a lock file fails
 */
public void releaseAll() throws Exception {
	Collection<String> children = getAllPaths();

	Exception exception = null;

	for (String child: children) {
		try {
			release(child);
		} catch (Exception e) {
			exception = ExceptionUtils.firstOrSuppressed(e, exception);
		}
	}

	if (exception != null) {
		throw new Exception("Could not properly release all state nodes.", exception);
	}
}
 

private void completeFuture(Object ignored, Throwable throwable) {
	synchronized (lock) {
		futuresCompleted++;

		if (throwable != null) {
			globalThrowable = ExceptionUtils.firstOrSuppressed(throwable, globalThrowable);
		}

		if (futuresCompleted == numFuturesTotal) {
			if (globalThrowable != null) {
				completeExceptionally(globalThrowable);
			} else {
				complete(null);
			}
		}
	}
}
 

/**
 * Called to shut down the TaskManager. The method closes all TaskManager services.
 */
@Override
public CompletableFuture<Void> onStop() {
	log.info("Stopping TaskExecutor {}.", getAddress());

	Throwable throwable = null;

	if (resourceManagerConnection != null) {
		resourceManagerConnection.close();
	}

	for (JobManagerConnection jobManagerConnection : jobManagerConnections.values()) {
		try {
			disassociateFromJobManager(jobManagerConnection, new FlinkException("The TaskExecutor is shutting down."));
		} catch (Throwable t) {
			throwable = ExceptionUtils.firstOrSuppressed(t, throwable);
		}
	}

	try {
		stopTaskExecutorServices();
	} catch (Exception e) {
		throwable = ExceptionUtils.firstOrSuppressed(e, throwable);
	}

	if (throwable != null) {
		return FutureUtils.completedExceptionally(new FlinkException("Error while shutting the TaskExecutor down.", throwable));
	} else {
		log.info("Stopped TaskExecutor {}.", getAddress());
		return CompletableFuture.completedFuture(null);
	}
}
 

public void startFlinkCluster() throws IOException {
	LOG.info("Starting Flink cluster.");
	AutoClosableProcess.runBlocking(bin.resolve("start-cluster.sh").toAbsolutePath().toString());

	final OkHttpClient client = new OkHttpClient();

	final Request request = new Request.Builder()
		.get()
		.url("http://localhost:8081/taskmanagers")
		.build();

	Exception reportedException = null;
	for (int retryAttempt = 0; retryAttempt < 30; retryAttempt++) {
		try (Response response = client.newCall(request).execute()) {
			if (response.isSuccessful()) {
				final String json = response.body().string();
				final JsonNode taskManagerList = OBJECT_MAPPER.readTree(json)
					.get("taskmanagers");

				if (taskManagerList != null && taskManagerList.size() > 0) {
					LOG.info("Dispatcher REST endpoint is up.");
					return;
				}
			}
		} catch (IOException ioe) {
			reportedException = ExceptionUtils.firstOrSuppressed(ioe, reportedException);
		}

		LOG.info("Waiting for dispatcher REST endpoint to come up...");
		try {
			Thread.sleep(1000);
		} catch (InterruptedException e) {
			Thread.currentThread().interrupt();
			reportedException = ExceptionUtils.firstOrSuppressed(e, reportedException);
		}
	}
	throw new AssertionError("Dispatcher REST endpoint did not start in time.", reportedException);
}
 

@Nonnull
@Override
public SnapshotResult<StreamStateHandle> closeAndFinalizeCheckpointStreamResult() throws IOException {

	final StreamStateHandle primaryStreamStateHandle;

	try {
		primaryStreamStateHandle = outputStream.closeAndGetPrimaryHandle();
	} catch (IOException primaryEx) {
		try {
			outputStream.close();
		} catch (IOException closeEx) {
			primaryEx = ExceptionUtils.firstOrSuppressed(closeEx, primaryEx);
		}
		throw primaryEx;
	}

	StreamStateHandle secondaryStreamStateHandle = null;

	try {
		secondaryStreamStateHandle = outputStream.closeAndGetSecondaryHandle();
	} catch (IOException secondaryEx) {
		LOG.warn("Exception from secondary/local checkpoint stream.", secondaryEx);
	}

	if (primaryStreamStateHandle != null) {
		if (secondaryStreamStateHandle != null) {
			return SnapshotResult.withLocalState(primaryStreamStateHandle, secondaryStreamStateHandle);
		} else {
			return SnapshotResult.of(primaryStreamStateHandle);
		}
	} else {
		return SnapshotResult.empty();
	}
}
 

@Nonnull
@Override
public SnapshotResult<StreamStateHandle> closeAndFinalizeCheckpointStreamResult() throws IOException {

	final StreamStateHandle primaryStreamStateHandle;

	try {
		primaryStreamStateHandle = outputStream.closeAndGetPrimaryHandle();
	} catch (IOException primaryEx) {
		try {
			outputStream.close();
		} catch (IOException closeEx) {
			primaryEx = ExceptionUtils.firstOrSuppressed(closeEx, primaryEx);
		}
		throw primaryEx;
	}

	StreamStateHandle secondaryStreamStateHandle = null;

	try {
		secondaryStreamStateHandle = outputStream.closeAndGetSecondaryHandle();
	} catch (IOException secondaryEx) {
		LOG.warn("Exception from secondary/local checkpoint stream.", secondaryEx);
	}

	if (primaryStreamStateHandle != null) {
		if (secondaryStreamStateHandle != null) {
			return SnapshotResult.withLocalState(primaryStreamStateHandle, secondaryStreamStateHandle);
		} else {
			return SnapshotResult.of(primaryStreamStateHandle);
		}
	} else {
		return SnapshotResult.empty();
	}
}
 

public void startFlinkCluster() throws IOException {
	AutoClosableProcess.runBlocking("Start Flink cluster", bin.resolve("start-cluster.sh").toAbsolutePath().toString());

	final OkHttpClient client = new OkHttpClient();

	final Request request = new Request.Builder()
		.get()
		.url("http://localhost:8081/taskmanagers")
		.build();

	Exception reportedException = null;
	for (int retryAttempt = 0; retryAttempt < 30; retryAttempt++) {
		try (Response response = client.newCall(request).execute()) {
			if (response.isSuccessful()) {
				final String json = response.body().string();
				final JsonNode taskManagerList = OBJECT_MAPPER.readTree(json)
					.get("taskmanagers");

				if (taskManagerList != null && taskManagerList.size() > 0) {
					LOG.info("Dispatcher REST endpoint is up.");
					return;
				}
			}
		} catch (IOException ioe) {
			reportedException = ExceptionUtils.firstOrSuppressed(ioe, reportedException);
		}

		LOG.info("Waiting for dispatcher REST endpoint to come up...");
		try {
			Thread.sleep(1000);
		} catch (InterruptedException e) {
			Thread.currentThread().interrupt();
			reportedException = ExceptionUtils.firstOrSuppressed(e, reportedException);
		}
	}
	throw new AssertionError("Dispatcher REST endpoint did not start in time.", reportedException);
}
 

private void ensureRunning() throws Exception {
	// this check should be performed *at least after* enqueuing a request
	// checking before is not enough because (check + enqueue) is not atomic
	if (wasClosed || !thread.isAlive()) {
		cleanupRequests();
		throw ExceptionUtils.firstOrSuppressed(new IllegalStateException("not running"), thrown);
	}
}
 

private void handleOnStopException(Throwable throwableBeforeTasksCompletion, Throwable throwableAfterTasksCompletion) {
	final Throwable throwable;

	if (throwableBeforeTasksCompletion != null) {
		throwable = ExceptionUtils.firstOrSuppressed(throwableBeforeTasksCompletion, throwableAfterTasksCompletion);
	} else {
		throwable = throwableAfterTasksCompletion;
	}

	if (throwable != null) {
		throw new CompletionException(new FlinkException("Error while shutting the TaskExecutor down.", throwable));
	} else {
		log.info("Stopped TaskExecutor {}.", getAddress());
	}
}
 

/**
 * Shuts down this registry and the associated {@link MetricReporter}.
 *
 * <p>NOTE: This operation is asynchronous and returns a future which is completed
 * once the shutdown operation has been completed.
 *
 * @return Future which is completed once the {@link MetricRegistryImpl}
 * is shut down.
 */
public CompletableFuture<Void> shutdown() {
	synchronized (lock) {
		if (isShutdown) {
			return terminationFuture;
		} else {
			isShutdown = true;
			final Collection<CompletableFuture<Void>> terminationFutures = new ArrayList<>(3);
			final Time gracePeriod = Time.seconds(1L);

			if (queryService != null) {
				final CompletableFuture<Void> queryServiceTerminationFuture = ActorUtils.nonBlockingShutDown(
					gracePeriod.toMilliseconds(),
					TimeUnit.MILLISECONDS,
					queryService);

				terminationFutures.add(queryServiceTerminationFuture);
			}

			Throwable throwable = null;
			for (MetricReporter reporter : reporters) {
				try {
					reporter.close();
				} catch (Throwable t) {
					throwable = ExceptionUtils.firstOrSuppressed(t, throwable);
				}
			}
			reporters.clear();

			if (throwable != null) {
				terminationFutures.add(
					FutureUtils.completedExceptionally(
						new FlinkException("Could not shut down the metric reporters properly.", throwable)));
			}

			final CompletableFuture<Void> executorShutdownFuture = ExecutorUtils.nonBlockingShutdown(
				gracePeriod.toMilliseconds(),
				TimeUnit.MILLISECONDS,
				executor);

			terminationFutures.add(executorShutdownFuture);

			FutureUtils
				.completeAll(terminationFutures)
				.whenComplete(
					(Void ignored, Throwable error) -> {
						if (error != null) {
							terminationFuture.completeExceptionally(error);
						} else {
							terminationFuture.complete(null);
						}
					});

			return terminationFuture;
		}
	}
}
 

/**
 * Shuts down the BLOB server.
 */
@Override
public void close() throws IOException {
	cleanupTimer.cancel();

	if (shutdownRequested.compareAndSet(false, true)) {
		Exception exception = null;

		try {
			this.serverSocket.close();
		}
		catch (IOException ioe) {
			exception = ioe;
		}

		// wake the thread up, in case it is waiting on some operation
		interrupt();

		try {
			join();
		}
		catch (InterruptedException ie) {
			Thread.currentThread().interrupt();

			LOG.debug("Error while waiting for this thread to die.", ie);
		}

		synchronized (activeConnections) {
			if (!activeConnections.isEmpty()) {
				for (BlobServerConnection conn : activeConnections) {
					LOG.debug("Shutting down connection {}.", conn.getName());
					conn.close();
				}
				activeConnections.clear();
			}
		}

		// Clean up the storage directory
		try {
			FileUtils.deleteDirectory(storageDir);
		}
		catch (IOException e) {
			exception = ExceptionUtils.firstOrSuppressed(e, exception);
		}

		// Remove shutdown hook to prevent resource leaks
		ShutdownHookUtil.removeShutdownHook(shutdownHook, getClass().getSimpleName(), LOG);

		if (LOG.isInfoEnabled()) {
			LOG.info("Stopped BLOB server at {}:{}", serverSocket.getInetAddress().getHostAddress(), getPort());
		}

		ExceptionUtils.tryRethrowIOException(exception);
	}
}
 

/**
 * Shuts down the BLOB server.
 */
@Override
public void close() throws IOException {
	cleanupTimer.cancel();

	if (shutdownRequested.compareAndSet(false, true)) {
		Exception exception = null;

		try {
			this.serverSocket.close();
		}
		catch (IOException ioe) {
			exception = ioe;
		}

		// wake the thread up, in case it is waiting on some operation
		interrupt();

		try {
			join();
		}
		catch (InterruptedException ie) {
			Thread.currentThread().interrupt();

			LOG.debug("Error while waiting for this thread to die.", ie);
		}

		synchronized (activeConnections) {
			if (!activeConnections.isEmpty()) {
				for (BlobServerConnection conn : activeConnections) {
					LOG.debug("Shutting down connection {}.", conn.getName());
					conn.close();
				}
				activeConnections.clear();
			}
		}

		// Clean up the storage directory
		try {
			FileUtils.deleteDirectory(storageDir);
		}
		catch (IOException e) {
			exception = ExceptionUtils.firstOrSuppressed(e, exception);
		}

		// Remove shutdown hook to prevent resource leaks
		ShutdownHookUtil.removeShutdownHook(shutdownHook, getClass().getSimpleName(), LOG);

		if (LOG.isInfoEnabled()) {
			LOG.info("Stopped BLOB server at {}:{}", serverSocket.getInetAddress().getHostAddress(), getPort());
		}

		ExceptionUtils.tryRethrowIOException(exception);
	}
}