org.apache.flink.util.ExceptionUtils Java Examples

@Override
public void onFailure(ActionRequest actionRequest, Throwable throwable, int i, RequestIndexer requestIndexer) throws Throwable {
    if (ExceptionUtils.findThrowable(throwable, EsRejectedExecutionException.class).isPresent()) {
        requestIndexer.add(new ActionRequest[]{actionRequest});
    } else {
        if (ExceptionUtils.findThrowable(throwable, SocketTimeoutException.class).isPresent()) {
            return;
        } else {
            Optional<IOException> exp = ExceptionUtils.findThrowable(throwable, IOException.class);
            if (exp.isPresent()) {
                IOException ioExp = exp.get();
                if (ioExp != null && ioExp.getMessage() != null && ioExp.getMessage().contains("max retry timeout")) {
                    log.error(ioExp.getMessage());
                    return;
                }
            }
        }
        throw throwable;
    }
}
 

/**
 * Tests that an querying an unknown trigger id will return an exceptionally completed
 * future.
 */
@Test
public void testUnknownTriggerId() throws Exception {
	final TestingRestfulGateway testingRestfulGateway = new TestingRestfulGateway.Builder().build();

	try {
		testingStatusHandler.handleRequest(
			statusOperationRequest(new TriggerId()),
			testingRestfulGateway).get();

		fail("This should have failed with a RestHandlerException.");
	} catch (ExecutionException ee) {
		final Optional<RestHandlerException> optionalRestHandlerException = ExceptionUtils.findThrowable(ee, RestHandlerException.class);

		assertThat(optionalRestHandlerException.isPresent(), is(true));

		final RestHandlerException restHandlerException = optionalRestHandlerException.get();

		assertThat(restHandlerException.getMessage(), containsString("Operation not found"));
		assertThat(restHandlerException.getHttpResponseStatus(), is(HttpResponseStatus.NOT_FOUND));
	}
}
 

/**
 * Sends a SavepointTriggerMessage to the job manager.
 */
private String triggerSavepoint(ClusterClient<?> clusterClient, JobID jobId, String savepointDirectory) throws FlinkException {
	logAndSysout("Triggering savepoint for job " + jobId + '.');
	CompletableFuture<String> savepointPathFuture = clusterClient.triggerSavepoint(jobId, savepointDirectory);

	logAndSysout("Waiting for response...");

	final String savepointPath;

	try {
		savepointPath = savepointPathFuture.get();
	}
	catch (Exception e) {
		Throwable cause = ExceptionUtils.stripExecutionException(e);
		throw new FlinkException("Triggering a savepoint for the job " + jobId + " failed.", cause);
	}

	logAndSysout("Savepoint completed. Path: " + savepointPath);
	logAndSysout("You can resume your program from this savepoint with the run command.");

	return savepointPath;
}
 

@Override
public void discardState() throws Exception {

	Exception aggregatedExceptions = null;

	if (jobManagerOwnedSnapshot != null) {
		try {
			jobManagerOwnedSnapshot.discardState();
		} catch (Exception remoteDiscardEx) {
			aggregatedExceptions = remoteDiscardEx;
		}
	}

	if (taskLocalSnapshot != null) {
		try {
			taskLocalSnapshot.discardState();
		} catch (Exception localDiscardEx) {
			aggregatedExceptions = ExceptionUtils.firstOrSuppressed(localDiscardEx, aggregatedExceptions);
		}
	}

	if (aggregatedExceptions != null) {
		throw aggregatedExceptions;
	}
}
 

/**
 * Loads the factories for the file systems directly supported by Flink.
 * Aside from the {@link LocalFileSystem}, these file systems are loaded
 * via Java's service framework.
 *
 * @return A map from the file system scheme to corresponding file system factory.
 */
private static List<FileSystemFactory> loadFileSystemFactories(
	Collection<Supplier<Iterator<FileSystemFactory>>> factoryIteratorsSuppliers) {

	final ArrayList<FileSystemFactory> list = new ArrayList<>();

	// by default, we always have the local file system factory
	list.add(new LocalFileSystemFactory());

	LOG.debug("Loading extension file systems via services");

	for (Supplier<Iterator<FileSystemFactory>> factoryIteratorsSupplier : factoryIteratorsSuppliers) {
		try {
			addAllFactoriesToList(factoryIteratorsSupplier.get(), list);
		} catch (Throwable t) {
			// catching Throwable here to handle various forms of class loading
			// and initialization errors
			ExceptionUtils.rethrowIfFatalErrorOrOOM(t);
			LOG.error("Failed to load additional file systems via services", t);
		}
	}

	return Collections.unmodifiableList(list);
}
 

/**
 * Sends a {@link org.apache.flink.runtime.messages.JobManagerMessages.TriggerSavepoint}
 * message to the job manager.
 */
private String triggerSavepoint(ClusterClient<?> clusterClient, JobID jobId, String savepointDirectory) throws FlinkException {
	logAndSysout("Triggering savepoint for job " + jobId + '.');
	CompletableFuture<String> savepointPathFuture = clusterClient.triggerSavepoint(jobId, savepointDirectory);

	logAndSysout("Waiting for response...");

	final String savepointPath;

	try {
		savepointPath = savepointPathFuture.get();
	}
	catch (Exception e) {
		Throwable cause = ExceptionUtils.stripExecutionException(e);
		throw new FlinkException("Triggering a savepoint for the job " + jobId + " failed.", cause);
	}

	logAndSysout("Savepoint completed. Path: " + savepointPath);
	logAndSysout("You can resume your program from this savepoint with the run command.");

	return savepointPath;
}
 

@Test
public void testKeyedMapStateRegistrationFailsIfNewStateSerializerIsIncompatible() {
	final String stateName = "test-name";

	try {
		testKeyedMapStateUpgrade(
			new MapStateDescriptor<>(
			stateName,
			IntSerializer.INSTANCE,
			new TestType.V1TestTypeSerializer()),
			new MapStateDescriptor<>(
			stateName,
			IntSerializer.INSTANCE,
			// restore with a V2 serializer that has a different schema
			new TestType.IncompatibleTestTypeSerializer()));
		fail("should have failed");
	} catch (Exception expected) {
		Assert.assertTrue(ExceptionUtils.findThrowable(expected, StateMigrationException.class).isPresent());
	}
}
 

/**
 * Test receive registration with unmatched leadershipId from job master.
 */
@Test
public void testRegisterJobMasterWithUnmatchedLeaderSessionId1() throws Exception {
	final ResourceManagerGateway wronglyFencedGateway = rpcService.connect(resourceManager.getAddress(), ResourceManagerId.generate(), ResourceManagerGateway.class)
		.get(TIMEOUT.toMilliseconds(), TimeUnit.MILLISECONDS);

	// test throw exception when receive a registration from job master which takes unmatched leaderSessionId
	CompletableFuture<RegistrationResponse> unMatchedLeaderFuture = wronglyFencedGateway.registerJobManager(
		jobMasterGateway.getFencingToken(),
		jobMasterResourceId,
		jobMasterGateway.getAddress(),
		jobId,
		TIMEOUT);

	try {
		unMatchedLeaderFuture.get(5L, TimeUnit.SECONDS);
		fail("Should fail because we are using the wrong fencing token.");
	} catch (ExecutionException e) {
		assertTrue(ExceptionUtils.stripExecutionException(e) instanceof FencingTokenException);
	}
}
 

@Test
public void testKeyedValueStateRegistrationFailsIfNewStateSerializerIsIncompatible() {
	final String stateName = "test-name";

	try {
		testKeyedValueStateUpgrade(
			new ValueStateDescriptor<>(
				stateName,
				new TestType.V1TestTypeSerializer()),
			new ValueStateDescriptor<>(
				stateName,
				new TestType.IncompatibleTestTypeSerializer()));

		fail("should have failed");
	} catch (Exception expected) {
		Assert.assertTrue(ExceptionUtils.findThrowable(expected, StateMigrationException.class).isPresent());
	}
}
 

@Test
public void testOperatorUnionListStateRegistrationFailsIfNewSerializerIsIncompatible() {
	final String stateName = "union-list-state";

	try {
		testOperatorUnionListStateUpgrade(
			new ListStateDescriptor<>(
				stateName,
				new TestType.V1TestTypeSerializer()),
			new ListStateDescriptor<>(
				stateName,
				// restore with a new incompatible serializer
				new TestType.IncompatibleTestTypeSerializer()));

		fail("should have failed.");
	} catch (Exception e) {
		Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent());
	}
}
 

/**
 * Tests that a retry future is failed after all retries have been consumed.
 */
@Test(expected = FutureUtils.RetryException.class)
public void testRetryFailure() throws Throwable {
	final int retries = 3;

	CompletableFuture<?> retryFuture = FutureUtils.retry(
		() -> FutureUtils.completedExceptionally(new FlinkException("Test exception")),
		retries,
		TestingUtils.defaultExecutor());

	try {
		retryFuture.get();
	} catch (ExecutionException ee) {
		throw ExceptionUtils.stripExecutionException(ee);
	}
}
 

/**
 * Resets the master hooks.
 *
 * @param hooks The hooks to reset
 *
 * @throws FlinkException Thrown, if the hooks throw an exception.
 */
public static void reset(
	final Collection<MasterTriggerRestoreHook<?>> hooks,
	@SuppressWarnings("unused") final Logger log) throws FlinkException {

	for (MasterTriggerRestoreHook<?> hook : hooks) {
		final String id = hook.getIdentifier();
		try {
			hook.reset();
		}
		catch (Throwable t) {
			ExceptionUtils.rethrowIfFatalErrorOrOOM(t);
			throw new FlinkException("Error while resetting checkpoint master hook '" + id + '\'', t);
		}
	}
}
 

/**
 * Releases all lock nodes of this ZooKeeperStateHandleStore.
 *
 * @throws Exception if the delete operation of a lock file fails
 */
public void releaseAll() throws Exception {
	Collection<String> children = getAllPaths();

	Exception exception = null;

	for (String child: children) {
		try {
			release(child);
		} catch (Exception e) {
			exception = ExceptionUtils.firstOrSuppressed(e, exception);
		}
	}

	if (exception != null) {
		throw new Exception("Could not properly release all state nodes.", exception);
	}
}
 

private boolean userClassPathContainsJobClass(String jobClassName) {
	for (URL userClassPath : userClassPaths) {
		try (final JarFile jarFile = new JarFile(userClassPath.getFile())) {
			if (jarContainsJobClass(jobClassName, jarFile)) {
				return true;
			}
		} catch (IOException e) {
			ExceptionUtils.rethrow(
				e,
				String.format(
					"Failed to open user class path %s. Make sure that all files on the user class path can be accessed.",
					userClassPath));
		}
	}
	return false;
}
 

@Test
public void testSupplyAsyncFailure() throws Exception {
	final String exceptionMessage = "Test exception";
	final FlinkException testException = new FlinkException(exceptionMessage);
	final CompletableFuture<Object> future = FutureUtils.supplyAsync(
		() -> {
			throw testException;
		},
		TestingUtils.defaultExecutor());

	try {
		future.get();
		fail("Expected an exception.");
	} catch (ExecutionException e) {
		assertThat(ExceptionUtils.findThrowableWithMessage(e, exceptionMessage).isPresent(), is(true));
	}
}
 

@Test
public void testConnectionTimeout() throws Exception {
	final Configuration config = new Configuration();
	config.setLong(RestOptions.CONNECTION_TIMEOUT, 1);
	try (final RestClient restClient = new RestClient(RestClientConfiguration.fromConfiguration(config), Executors.directExecutor())) {
		restClient.sendRequest(
			unroutableIp,
			80,
			new TestMessageHeaders(),
			EmptyMessageParameters.getInstance(),
			EmptyRequestBody.getInstance())
			.get(60, TimeUnit.SECONDS);
	} catch (final ExecutionException e) {
		final Throwable throwable = ExceptionUtils.stripExecutionException(e);
		assertThat(throwable, instanceOf(ConnectTimeoutException.class));
		assertThat(throwable.getMessage(), containsString(unroutableIp));
	}
}
 

/**
 * Polls the next element from the Handover, possibly blocking until the next element is
 * available. This method behaves similar to polling from a blocking queue.
 *
 * <p>If an exception was handed in by the producer ({@link #reportError(Throwable)}), then
 * that exception is thrown rather than an element being returned.
 *
 * @return The next element (buffer of records, never null).
 *
 * @throws ClosedException Thrown if the Handover was {@link #close() closed}.
 * @throws Exception Rethrows exceptions from the {@link #reportError(Throwable)} method.
 */
@Nonnull
public ConsumerRecords<byte[], byte[]> pollNext() throws Exception {
	synchronized (lock) {
		while (next == null && error == null) {
			lock.wait();
		}

		ConsumerRecords<byte[], byte[]> n = next;
		if (n != null) {
			next = null;
			lock.notifyAll();
			return n;
		}
		else {
			ExceptionUtils.rethrowException(error, error.getMessage());

			// this statement cannot be reached since the above method always throws an exception
			// this is only here to silence the compiler and any warnings
			return ConsumerRecords.empty();
		}
	}
}
 

private static boolean truncate(final FileSystem hadoopFs, final Path file, final long length) throws IOException {
	if (!HadoopUtils.isMinHadoopVersion(2, 7)) {
		throw new IllegalStateException("Truncation is not available in hadoop version < 2.7 , You are on Hadoop " + VersionInfo.getVersion());
	}

	if (truncateHandle != null) {
		try {
			return (Boolean) truncateHandle.invoke(hadoopFs, file, length);
		}
		catch (InvocationTargetException e) {
			ExceptionUtils.rethrowIOException(e.getTargetException());
		}
		catch (Throwable t) {
			throw new IOException(
					"Truncation of file failed because of access/linking problems with Hadoop's truncate call. " +
							"This is most likely a dependency conflict or class loading problem.");
		}
	}
	else {
		throw new IllegalStateException("Truncation handle has not been initialized");
	}
	return false;
}
 

/**
 * Tests that the {@link Dispatcher} terminates if it cannot recover jobs ids from
 * the {@link SubmittedJobGraphStore}. See FLINK-8943.
 */
@Test
public void testFatalErrorAfterJobIdRecoveryFailure() throws Exception {
	dispatcher = createAndStartDispatcher(heartbeatServices, haServices, new ExpectedJobIdJobManagerRunnerFactory(TEST_JOB_ID, createdJobManagerRunnerLatch));

	final FlinkException testException = new FlinkException("Test exception");
	submittedJobGraphStore.setJobIdsFunction(
		(Collection<JobID> jobIds) -> {
			throw testException;
		});

	electDispatcher();

	// we expect that a fatal error occurred
	final Throwable error = fatalErrorHandler.getErrorFuture().get(TIMEOUT.toMilliseconds(), TimeUnit.MILLISECONDS);

	assertThat(ExceptionUtils.findThrowableWithMessage(error, testException.getMessage()).isPresent(), is(true));

	fatalErrorHandler.clearError();
}
 

@Test
public void testKeyedValueStateRegistrationFailsIfNewStateSerializerIsIncompatible() {
	final String stateName = "test-name";

	try {
		testKeyedValueStateUpgrade(
			new ValueStateDescriptor<>(
				stateName,
				new TestType.V1TestTypeSerializer()),
			new ValueStateDescriptor<>(
				stateName,
				new TestType.IncompatibleTestTypeSerializer()));

		fail("should have failed");
	} catch (Exception expected) {
		Assert.assertTrue(ExceptionUtils.findThrowable(expected, StateMigrationException.class).isPresent());
	}
}
 

/**
 * This test validates that the factory can be instantiated and configured even
 * when MapR and Hadoop classes are missing from the classpath.
 */
@Test
public void testInstantiationWithoutMapRClasses() throws Exception {
	// we do reflection magic here to instantiate the test in another class
	// loader, to make sure no MapR and Hadoop classes are in the classpath

	final String testClassName = "org.apache.flink.runtime.fs.maprfs.MapRFreeTests";

	URLClassLoader parent = (URLClassLoader) getClass().getClassLoader();
	ClassLoader maprFreeClassLoader = new MapRFreeClassLoader(parent);
	Class<?> testClass = Class.forName(testClassName, false, maprFreeClassLoader);
	Method m = testClass.getDeclaredMethod("test");

	try {
		m.invoke(null);
	}
	catch (InvocationTargetException e) {
		ExceptionUtils.rethrowException(e.getTargetException(), "exception in method");
	}
}
 

@Override
public void run() {
	signalRunLatch.trigger();
	continueRunLatch.countDown();

	try {
		// poor man's barrier because it can happen that the async operations thread gets
		// interrupted by the mail box thread. The CyclicBarrier would in this case fail
		// all participants of the barrier, leaving the future uncompleted
		continueRunLatch.await();
	} catch (InterruptedException e) {
		ExceptionUtils.rethrow(e);
	}

	future.complete(value);
}
 

private CompletableFuture<Acknowledge> internalSubmitJob(JobGraph jobGraph) {
	log.info("Submitting job {} ({}).", jobGraph.getJobID(), jobGraph.getName());

	final CompletableFuture<Acknowledge> persistAndRunFuture = waitForTerminatingJobManager(jobGraph.getJobID(), jobGraph, this::persistAndRunJob)
		.thenApply(ignored -> Acknowledge.get());

	return persistAndRunFuture.handleAsync((acknowledge, throwable) -> {
		if (throwable != null) {
			cleanUpJobData(jobGraph.getJobID(), true);

			final Throwable strippedThrowable = ExceptionUtils.stripCompletionException(throwable);
			log.error("Failed to submit job {}.", jobGraph.getJobID(), strippedThrowable);
			throw new CompletionException(
				new JobSubmissionException(jobGraph.getJobID(), "Failed to submit job.", strippedThrowable));
		} else {
			return acknowledge;
		}
	}, getRpcService().getExecutor());
}
 

@Override
public void broadcastEvent(AbstractEvent event) throws IOException {
	IOException exception = null;
	for (RecordWriter recordWriter : recordWriters) {
		try {
			recordWriter.broadcastEvent(event);
		} catch (IOException e) {
			exception = ExceptionUtils.firstOrSuppressed(
				new IOException("Could not send event to downstream tasks.", e), exception);
		}
	}

	if (exception != null) {
		throw exception;
	}
}
 

@Test
public void testUnknownJobId() throws Exception {
	// test unknown job Id
	JobID jid = new JobID();

	String[] parameters = { "-p", "test-target-dir", jid.toString() };
	String expectedMessage = "Test exception";
	FlinkException testException = new FlinkException(expectedMessage);
	final ClusterClient<String> clusterClient = createClusterClient(testException);
	MockedCliFrontend testFrontend = new MockedCliFrontend(clusterClient);

	try {
		testFrontend.stop(parameters);
		fail("Should have failed.");
	} catch (FlinkException e) {
		assertTrue(ExceptionUtils.findThrowableWithMessage(e, expectedMessage).isPresent());
	}
}
 

@Test
public void testOperatorUnionListStateRegistrationFailsIfNewSerializerIsIncompatible() {
	final String stateName = "union-list-state";

	try {
		testOperatorUnionListStateUpgrade(
			new ListStateDescriptor<>(
				stateName,
				new TestType.V1TestTypeSerializer()),
			new ListStateDescriptor<>(
				stateName,
				// restore with a new incompatible serializer
				new TestType.IncompatibleTestTypeSerializer()));

		fail("should have failed.");
	} catch (Exception e) {
		Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent());
	}
}
 

@Test
public void testFailedDelete() throws Exception {
	makeJarDirReadOnly();

	final HandlerRequest<EmptyRequestBody, JarDeleteMessageParameters> request = createRequest(TEST_JAR_NAME);
	try {
		jarDeleteHandler.handleRequest(request, restfulGateway).get();
	} catch (final ExecutionException e) {
		final Throwable throwable = ExceptionUtils.stripCompletionException(e.getCause());
		assertThat(throwable, instanceOf(RestHandlerException.class));

		final RestHandlerException restHandlerException = (RestHandlerException) throwable;
		assertThat(restHandlerException.getMessage(), containsString("Failed to delete jar"));
		assertThat(restHandlerException.getHttpResponseStatus(), equalTo(HttpResponseStatus.INTERNAL_SERVER_ERROR));
	}
}
 

private T attemptCreateAndRestore(Collection<S> restoreState) throws Exception {

		// create a new backend with necessary initialization.
		final T backendInstance = instanceSupplier.apply(restoreState);

		try {
			// register the backend with the registry to participate in task lifecycle w.r.t. cancellation.
			backendCloseableRegistry.registerCloseable(backendInstance);
			return backendInstance;
		} catch (Exception ex) {
			// dispose the backend, e.g. to release native resources, if failed to register it into registry.
			try {
				backendInstance.dispose();
			} catch (Exception disposeEx) {
				ex = ExceptionUtils.firstOrSuppressed(disposeEx, ex);
			}

			throw ex;
		}
	}
 

/**
 * Shutdown the {@link JobMaster} services.
 *
 * <p>This method makes sure all services are closed or shut down, even when an exception occurred
 * in the shutdown of one component. The first encountered exception is thrown, with successive
 * exceptions added as suppressed exceptions.
 *
 * @throws Exception The first Exception encountered during shutdown.
 */
public void shutdown() throws Exception {
	Throwable firstException = null;

	try {
		scheduledExecutorService.shutdownNow();
	} catch (Throwable t) {
		firstException = t;
	}

	libraryCacheManager.shutdown();
	stackTraceSampleCoordinator.shutDown();
	backPressureStatsTracker.shutDown();

	if (firstException != null) {
		ExceptionUtils.rethrowException(firstException, "Error while shutting down JobManager services");
	}
}
 

@Test
	public void testTriggerCheckpointWithShuttingDownCoordinator() throws Exception {
		// set up the coordinator and validate the initial state
		CheckpointCoordinator checkpointCoordinator = createCheckpointCoordinator();
checkpointCoordinator.startCheckpointScheduler();
		final CompletableFuture<CompletedCheckpoint> onCompletionPromise =
			triggerPeriodicCheckpoint(checkpointCoordinator);

		checkpointCoordinator.shutdown(JobStatus.FAILED);
		manuallyTriggeredScheduledExecutor.triggerAll();
		try {
			onCompletionPromise.get();
			fail("Should not reach here");
		} catch (ExecutionException e) {
			final Optional<CheckpointException> checkpointExceptionOptional =
				ExceptionUtils.findThrowable(e, CheckpointException.class);
			assertTrue(checkpointExceptionOptional.isPresent());
			assertEquals(CheckpointFailureReason.CHECKPOINT_COORDINATOR_SHUTDOWN,
				checkpointExceptionOptional.get().getCheckpointFailureReason());
		}
	}