org.apache.flink.runtime.execution.librarycache.FlinkUserCodeClassLoaders Java Examples

@Test
public void testKryoInChildClasspath() throws Exception {
	final Class<?> avroClass = AvroKryoSerializerUtils.class;

	final URL avroLocation = avroClass.getProtectionDomain().getCodeSource().getLocation();
	final URL kryoLocation = Kryo.class.getProtectionDomain().getCodeSource().getLocation();

	final ClassLoader parentClassLoader = new FilteredClassLoader(
			avroClass.getClassLoader(), AvroKryoSerializerUtils.class.getName());

	final ClassLoader userAppClassLoader = FlinkUserCodeClassLoaders.childFirst(
			new URL[] { avroLocation, kryoLocation },
			parentClassLoader,
			CoreOptions.ALWAYS_PARENT_FIRST_LOADER_PATTERNS.defaultValue().split(";"));

	final Class<?> userLoadedAvroClass = Class.forName(avroClass.getName(), false, userAppClassLoader);
	assertNotEquals(avroClass, userLoadedAvroClass);

	// call the 'addAvroGenericDataArrayRegistration(...)' method
	final Method m = userLoadedAvroClass.getMethod("addAvroGenericDataArrayRegistration", LinkedHashMap.class);

	final LinkedHashMap<String, ?> map = new LinkedHashMap<>();
	m.invoke(userLoadedAvroClass.newInstance(), map);

	assertEquals(1, map.size());
}
 

@BeforeClass
public static void setupClass() {
	libraryCacheManager = new BlobLibraryCacheManager(
		FailingPermanentBlobService.INSTANCE,
		FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST,
		new String[]{});

	defaultJobMasterServiceFactory = new TestingJobMasterServiceFactory();

	final JobVertex jobVertex = new JobVertex("Test vertex");
	jobVertex.setInvokableClass(NoOpInvokable.class);
	jobGraph = new JobGraph(jobVertex);

	archivedExecutionGraph = new ArchivedExecutionGraphBuilder()
		.setJobID(jobGraph.getJobID())
		.setState(JobStatus.FINISHED)
		.build();
}
 

@Test
public void testKryoInChildClasspath() throws Exception {
	final Class<?> avroClass = AvroKryoSerializerUtils.class;

	final URL avroLocation = avroClass.getProtectionDomain().getCodeSource().getLocation();
	final URL kryoLocation = Kryo.class.getProtectionDomain().getCodeSource().getLocation();

	final ClassLoader parentClassLoader = new FilteredClassLoader(
			avroClass.getClassLoader(), AvroKryoSerializerUtils.class.getName());

	final ClassLoader userAppClassLoader = FlinkUserCodeClassLoaders.childFirst(
			new URL[] { avroLocation, kryoLocation },
			parentClassLoader,
			CoreOptions.ALWAYS_PARENT_FIRST_LOADER_PATTERNS.defaultValue().split(";"));

	final Class<?> userLoadedAvroClass = Class.forName(avroClass.getName(), false, userAppClassLoader);
	assertNotEquals(avroClass, userLoadedAvroClass);

	// call the 'addAvroGenericDataArrayRegistration(...)' method
	final Method m = userLoadedAvroClass.getMethod("addAvroGenericDataArrayRegistration", LinkedHashMap.class);

	final LinkedHashMap<String, ?> map = new LinkedHashMap<>();
	m.invoke(userLoadedAvroClass.newInstance(), map);

	assertEquals(1, map.size());
}
 

@Test
public void testLibraryCacheManagerRegistration() throws Exception {
	final BlobLibraryCacheManager libraryCacheManager = new BlobLibraryCacheManager(
		VoidPermanentBlobService.INSTANCE,
		FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST,
		new String[]{});
	final JobManagerRunner jobManagerRunner = createJobManagerRunner(libraryCacheManager);

	try {
		jobManagerRunner.start();

		final JobID jobID = jobGraph.getJobID();
		assertThat(libraryCacheManager.hasClassLoader(jobID), is(true));

		jobManagerRunner.close();

		assertThat(libraryCacheManager.hasClassLoader(jobID), is(false));
	} finally {
		jobManagerRunner.close();
	}
}
 

@Test
public void testParentFirstClassLoading() throws Exception {
	final ClassLoader parentClassLoader = getClass().getClassLoader();

	// collect the libraries / class folders with RocksDB related code: the state backend and RocksDB itself
	final URL childCodePath = getClass().getProtectionDomain().getCodeSource().getLocation();

	final URLClassLoader childClassLoader1 = FlinkUserCodeClassLoaders.parentFirst(
			new URL[] { childCodePath }, parentClassLoader);

	final URLClassLoader childClassLoader2 = FlinkUserCodeClassLoaders.parentFirst(
			new URL[] { childCodePath }, parentClassLoader);

	final String className = ClassLoaderTest.class.getName();

	final Class<?> clazz1 = Class.forName(className, false, parentClassLoader);
	final Class<?> clazz2 = Class.forName(className, false, childClassLoader1);
	final Class<?> clazz3 = Class.forName(className, false, childClassLoader2);

	assertEquals(clazz1, clazz2);
	assertEquals(clazz1, clazz3);

	childClassLoader1.close();
	childClassLoader2.close();
}
 

@Test
public void testChildFirstClassLoading() throws Exception {
	final ClassLoader parentClassLoader = getClass().getClassLoader();

	// collect the libraries / class folders with RocksDB related code: the state backend and RocksDB itself
	final URL childCodePath = getClass().getProtectionDomain().getCodeSource().getLocation();

	final URLClassLoader childClassLoader1 = FlinkUserCodeClassLoaders.childFirst(
			new URL[] { childCodePath }, parentClassLoader, new String[0]);

	final URLClassLoader childClassLoader2 = FlinkUserCodeClassLoaders.childFirst(
			new URL[] { childCodePath }, parentClassLoader, new String[0]);

	final String className = ClassLoaderTest.class.getName();

	final Class<?> clazz1 = Class.forName(className, false, parentClassLoader);
	final Class<?> clazz2 = Class.forName(className, false, childClassLoader1);
	final Class<?> clazz3 = Class.forName(className, false, childClassLoader2);

	assertNotEquals(clazz1, clazz2);
	assertNotEquals(clazz1, clazz3);
	assertNotEquals(clazz2, clazz3);

	childClassLoader1.close();
	childClassLoader2.close();
}
 

@Test
public void testRepeatedChildFirstClassLoading() throws Exception {
	final ClassLoader parentClassLoader = getClass().getClassLoader();

	// collect the libraries / class folders with RocksDB related code: the state backend and RocksDB itself
	final URL childCodePath = getClass().getProtectionDomain().getCodeSource().getLocation();

	final URLClassLoader childClassLoader = FlinkUserCodeClassLoaders.childFirst(
			new URL[] { childCodePath }, parentClassLoader, new String[0]);

	final String className = ClassLoaderTest.class.getName();

	final Class<?> clazz1 = Class.forName(className, false, parentClassLoader);
	final Class<?> clazz2 = Class.forName(className, false, childClassLoader);
	final Class<?> clazz3 = Class.forName(className, false, childClassLoader);
	final Class<?> clazz4 = Class.forName(className, false, childClassLoader);

	assertNotEquals(clazz1, clazz2);

	assertEquals(clazz2, clazz3);
	assertEquals(clazz2, clazz4);

	childClassLoader.close();
}
 

@Test
public void testRepeatedParentFirstPatternClass() throws Exception {
	final String className = ClassLoaderTest.class.getName();
	final String parentFirstPattern = className.substring(0, className.lastIndexOf('.'));

	final ClassLoader parentClassLoader = getClass().getClassLoader();

	// collect the libraries / class folders with RocksDB related code: the state backend and RocksDB itself
	final URL childCodePath = getClass().getProtectionDomain().getCodeSource().getLocation();

	final URLClassLoader childClassLoader = FlinkUserCodeClassLoaders.childFirst(
			new URL[] { childCodePath }, parentClassLoader, new String[] { parentFirstPattern });

	final Class<?> clazz1 = Class.forName(className, false, parentClassLoader);
	final Class<?> clazz2 = Class.forName(className, false, childClassLoader);
	final Class<?> clazz3 = Class.forName(className, false, childClassLoader);
	final Class<?> clazz4 = Class.forName(className, false, childClassLoader);

	assertEquals(clazz1, clazz2);
	assertEquals(clazz1, clazz3);
	assertEquals(clazz1, clazz4);

	childClassLoader.close();
}
 

@Test
public void testRepeatedParentFirstPatternClass() throws Exception {
	final String className = ClassLoaderTest.class.getName();
	final String parentFirstPattern = className.substring(0, className.lastIndexOf('.'));

	final ClassLoader parentClassLoader = getClass().getClassLoader();

	// collect the libraries / class folders with RocksDB related code: the state backend and RocksDB itself
	final URL childCodePath = getClass().getProtectionDomain().getCodeSource().getLocation();

	final URLClassLoader childClassLoader = FlinkUserCodeClassLoaders.childFirst(
			new URL[] { childCodePath }, parentClassLoader, new String[] { parentFirstPattern });

	final Class<?> clazz1 = Class.forName(className, false, parentClassLoader);
	final Class<?> clazz2 = Class.forName(className, false, childClassLoader);
	final Class<?> clazz3 = Class.forName(className, false, childClassLoader);
	final Class<?> clazz4 = Class.forName(className, false, childClassLoader);

	assertEquals(clazz1, clazz2);
	assertEquals(clazz1, clazz3);
	assertEquals(clazz1, clazz4);

	childClassLoader.close();
}
 

@Test
public void testRepeatedChildFirstClassLoading() throws Exception {
	final ClassLoader parentClassLoader = getClass().getClassLoader();

	// collect the libraries / class folders with RocksDB related code: the state backend and RocksDB itself
	final URL childCodePath = getClass().getProtectionDomain().getCodeSource().getLocation();

	final URLClassLoader childClassLoader = FlinkUserCodeClassLoaders.childFirst(
			new URL[] { childCodePath }, parentClassLoader, new String[0]);

	final String className = ClassLoaderTest.class.getName();

	final Class<?> clazz1 = Class.forName(className, false, parentClassLoader);
	final Class<?> clazz2 = Class.forName(className, false, childClassLoader);
	final Class<?> clazz3 = Class.forName(className, false, childClassLoader);
	final Class<?> clazz4 = Class.forName(className, false, childClassLoader);

	assertNotEquals(clazz1, clazz2);

	assertEquals(clazz2, clazz3);
	assertEquals(clazz2, clazz4);

	childClassLoader.close();
}
 

@Test
public void testChildFirstClassLoading() throws Exception {
	final ClassLoader parentClassLoader = getClass().getClassLoader();

	// collect the libraries / class folders with RocksDB related code: the state backend and RocksDB itself
	final URL childCodePath = getClass().getProtectionDomain().getCodeSource().getLocation();

	final URLClassLoader childClassLoader1 = FlinkUserCodeClassLoaders.childFirst(
			new URL[] { childCodePath }, parentClassLoader, new String[0]);

	final URLClassLoader childClassLoader2 = FlinkUserCodeClassLoaders.childFirst(
			new URL[] { childCodePath }, parentClassLoader, new String[0]);

	final String className = ClassLoaderTest.class.getName();

	final Class<?> clazz1 = Class.forName(className, false, parentClassLoader);
	final Class<?> clazz2 = Class.forName(className, false, childClassLoader1);
	final Class<?> clazz3 = Class.forName(className, false, childClassLoader2);

	assertNotEquals(clazz1, clazz2);
	assertNotEquals(clazz1, clazz3);
	assertNotEquals(clazz2, clazz3);

	childClassLoader1.close();
	childClassLoader2.close();
}
 

@Test
public void testParentFirstClassLoading() throws Exception {
	final ClassLoader parentClassLoader = getClass().getClassLoader();

	// collect the libraries / class folders with RocksDB related code: the state backend and RocksDB itself
	final URL childCodePath = getClass().getProtectionDomain().getCodeSource().getLocation();

	final URLClassLoader childClassLoader1 = FlinkUserCodeClassLoaders.parentFirst(
			new URL[] { childCodePath }, parentClassLoader);

	final URLClassLoader childClassLoader2 = FlinkUserCodeClassLoaders.parentFirst(
			new URL[] { childCodePath }, parentClassLoader);

	final String className = ClassLoaderTest.class.getName();

	final Class<?> clazz1 = Class.forName(className, false, parentClassLoader);
	final Class<?> clazz2 = Class.forName(className, false, childClassLoader1);
	final Class<?> clazz3 = Class.forName(className, false, childClassLoader2);

	assertEquals(clazz1, clazz2);
	assertEquals(clazz1, clazz3);

	childClassLoader1.close();
	childClassLoader2.close();
}
 

@Test
public void testLibraryCacheManagerRegistration() throws Exception {
	final BlobLibraryCacheManager libraryCacheManager = new BlobLibraryCacheManager(
		VoidPermanentBlobService.INSTANCE,
		FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST,
		new String[]{});
	final JobManagerRunner jobManagerRunner = createJobManagerRunner(libraryCacheManager);

	try {
		jobManagerRunner.start();

		final JobID jobID = jobGraph.getJobID();
		assertThat(libraryCacheManager.hasClassLoader(jobID), is(true));

		jobManagerRunner.close();

		assertThat(libraryCacheManager.hasClassLoader(jobID), is(false));
	} finally {
		jobManagerRunner.close();
	}
}
 

@BeforeClass
public static void setupClass() {
	libraryCacheManager = new BlobLibraryCacheManager(
		FailingPermanentBlobService.INSTANCE,
		FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST,
		new String[]{});

	defaultJobMasterServiceFactory = new TestingJobMasterServiceFactory();

	final JobVertex jobVertex = new JobVertex("Test vertex");
	jobVertex.setInvokableClass(NoOpInvokable.class);
	jobGraph = new JobGraph(jobVertex);

	archivedExecutionGraph = new ArchivedExecutionGraphBuilder()
		.setJobID(jobGraph.getJobID())
		.setState(JobStatus.FINISHED)
		.build();
}
 

public static ClassLoader buildUserCodeClassLoader(List<URL> jars, List<URL> classpaths, ClassLoader parent) {
	URL[] urls = new URL[jars.size() + classpaths.size()];
	for (int i = 0; i < jars.size(); i++) {
		urls[i] = jars.get(i);
	}
	for (int i = 0; i < classpaths.size(); i++) {
		urls[i + jars.size()] = classpaths.get(i);
	}
	return FlinkUserCodeClassLoaders.parentFirst(urls, parent);
}
 

private static void setDefaultContextClassLoaderIfAbsent() {
  ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
  if (classLoader == null) {
    URLClassLoader flinkClassLoader =
        FlinkUserCodeClassLoaders.parentFirst(
            new URL[0], StatefulFunctionsJob.class.getClassLoader());
    Thread.currentThread().setContextClassLoader(flinkClassLoader);
  }
}
 

private static void setDefaultContextClassLoaderIfAbsent() {
  ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
  if (classLoader == null) {
    URLClassLoader flinkClassLoader =
        FlinkUserCodeClassLoaders.parentFirst(
            new URL[0], StatefulFunctionsJob.class.getClassLoader());
    Thread.currentThread().setContextClassLoader(flinkClassLoader);
  }
}
 

@Test
public void testKryoInChildClasspath() throws Exception {
	final Class<?> avroClass = AvroKryoSerializerUtils.class;

	final URL avroLocation = avroClass.getProtectionDomain().getCodeSource().getLocation();
	final URL kryoLocation = Kryo.class.getProtectionDomain().getCodeSource().getLocation();

	final ClassLoader parentClassLoader = new FilteredClassLoader(
			avroClass.getClassLoader(), AvroKryoSerializerUtils.class.getName());

	final ClassLoader userAppClassLoader = FlinkUserCodeClassLoaders.childFirst(
			new URL[] { avroLocation, kryoLocation },
			parentClassLoader,
			CoreOptions.ALWAYS_PARENT_FIRST_LOADER_PATTERNS.defaultValue().split(";"),
		NOOP_EXCEPTION_HANDLER);

	final Class<?> userLoadedAvroClass = Class.forName(avroClass.getName(), false, userAppClassLoader);
	assertNotEquals(avroClass, userLoadedAvroClass);

	// call the 'addAvroGenericDataArrayRegistration(...)' method
	final Method m = userLoadedAvroClass.getMethod("addAvroGenericDataArrayRegistration", LinkedHashMap.class);

	final LinkedHashMap<String, ?> map = new LinkedHashMap<>();
	m.invoke(userLoadedAvroClass.newInstance(), map);

	assertEquals(1, map.size());
}
 

@Test
public void testTwoSeparateClassLoaders() throws Exception {
	// collect the libraries / class folders with RocksDB related code: the state backend and RocksDB itself
	final URL codePath1 = RocksDBStateBackend.class.getProtectionDomain().getCodeSource().getLocation();
	final URL codePath2 = RocksDB.class.getProtectionDomain().getCodeSource().getLocation();

	final ClassLoader parent = getClass().getClassLoader();
	final ClassLoader loader1 = FlinkUserCodeClassLoaders.childFirst(new URL[] { codePath1, codePath2 }, parent, new String[0], NOOP_EXCEPTION_HANDLER);
	final ClassLoader loader2 = FlinkUserCodeClassLoaders.childFirst(new URL[] { codePath1, codePath2 }, parent, new String[0], NOOP_EXCEPTION_HANDLER);

	final String className = RocksDBStateBackend.class.getName();

	final Class<?> clazz1 = Class.forName(className, false, loader1);
	final Class<?> clazz2 = Class.forName(className, false, loader2);
	assertNotEquals("Test broken - the two reflectively loaded classes are equal", clazz1, clazz2);

	final Object instance1 = clazz1.getConstructor(String.class).newInstance(tmp.newFolder().toURI().toString());
	final Object instance2 = clazz2.getConstructor(String.class).newInstance(tmp.newFolder().toURI().toString());

	final String tempDir = tmp.newFolder().getAbsolutePath();

	final Method meth1 = clazz1.getDeclaredMethod("ensureRocksDBIsLoaded", String.class);
	final Method meth2 = clazz2.getDeclaredMethod("ensureRocksDBIsLoaded", String.class);
	meth1.setAccessible(true);
	meth2.setAccessible(true);

	// if all is well, these methods can both complete successfully
	meth1.invoke(instance1, tempDir);
	meth2.invoke(instance2, tempDir);
}
 

public TaskManagerConfiguration(
		int numberSlots,
		String[] tmpDirectories,
		Time timeout,
		@Nullable Time maxRegistrationDuration,
		Time initialRegistrationPause,
		Time maxRegistrationPause,
		Time refusedRegistrationPause,
		Configuration configuration,
		boolean exitJvmOnOutOfMemory,
		FlinkUserCodeClassLoaders.ResolveOrder classLoaderResolveOrder,
		String[] alwaysParentFirstLoaderPatterns,
		@Nullable String taskManagerLogPath,
		@Nullable String taskManagerStdoutPath,
		RetryingRegistrationConfiguration retryingRegistrationConfiguration) {

	this.numberSlots = numberSlots;
	this.tmpDirectories = Preconditions.checkNotNull(tmpDirectories);
	this.timeout = Preconditions.checkNotNull(timeout);
	this.maxRegistrationDuration = maxRegistrationDuration;
	this.initialRegistrationPause = Preconditions.checkNotNull(initialRegistrationPause);
	this.maxRegistrationPause = Preconditions.checkNotNull(maxRegistrationPause);
	this.refusedRegistrationPause = Preconditions.checkNotNull(refusedRegistrationPause);
	this.configuration = new UnmodifiableConfiguration(Preconditions.checkNotNull(configuration));
	this.exitJvmOnOutOfMemory = exitJvmOnOutOfMemory;
	this.classLoaderResolveOrder = classLoaderResolveOrder;
	this.alwaysParentFirstLoaderPatterns = alwaysParentFirstLoaderPatterns;
	this.taskManagerLogPath = taskManagerLogPath;
	this.taskManagerStdoutPath = taskManagerStdoutPath;
	this.retryingRegistrationConfiguration = retryingRegistrationConfiguration;
}
 

public ExecutionContext(Environment defaultEnvironment, SessionContext sessionContext, List<URL> dependencies,
		Configuration flinkConfig, Options commandLineOptions, List<CustomCommandLine<?>> availableCommandLines) {
	this.sessionContext = sessionContext.copy(); // create internal copy because session context is mutable
	this.mergedEnv = Environment.merge(defaultEnvironment, sessionContext.getEnvironment());
	this.dependencies = dependencies;
	this.flinkConfig = flinkConfig;

	// create class loader
	classLoader = FlinkUserCodeClassLoaders.parentFirst(
		dependencies.toArray(new URL[dependencies.size()]),
		this.getClass().getClassLoader());

	// create table sources & sinks.
	tableSources = new HashMap<>();
	tableSinks = new HashMap<>();
	mergedEnv.getTables().forEach((name, entry) -> {
		if (entry instanceof SourceTableEntry || entry instanceof SourceSinkTableEntry) {
			tableSources.put(name, createTableSource(mergedEnv.getExecution(), entry.asMap(), classLoader));
		}
		if (entry instanceof SinkTableEntry || entry instanceof SourceSinkTableEntry) {
			tableSinks.put(name, createTableSink(mergedEnv.getExecution(), entry.asMap(), classLoader));
		}
	});

	// create user-defined functions
	functions = new HashMap<>();
	mergedEnv.getFunctions().forEach((name, entry) -> {
		final UserDefinedFunction function = FunctionService.createFunction(entry.getDescriptor(), classLoader, false);
		functions.put(name, function);
	});

	// convert deployment options into command line options that describe a cluster
	commandLine = createCommandLine(mergedEnv.getDeployment(), commandLineOptions);
	activeCommandLine = findActiveCommandLine(availableCommandLines, commandLine);
	runOptions = createRunOptions(commandLine);
	clusterId = activeCommandLine.getClusterId(commandLine);
	clusterSpec = createClusterSpecification(activeCommandLine, commandLine);
}
 

@Test
public void testTwoSeparateClassLoaders() throws Exception {
	// collect the libraries / class folders with RocksDB related code: the state backend and RocksDB itself
	final URL codePath1 = RocksDBStateBackend.class.getProtectionDomain().getCodeSource().getLocation();
	final URL codePath2 = RocksDB.class.getProtectionDomain().getCodeSource().getLocation();

	final ClassLoader parent = getClass().getClassLoader();
	final ClassLoader loader1 = FlinkUserCodeClassLoaders.childFirst(new URL[] { codePath1, codePath2 }, parent, new String[0]);
	final ClassLoader loader2 = FlinkUserCodeClassLoaders.childFirst(new URL[] { codePath1, codePath2 }, parent, new String[0]);

	final String className = RocksDBStateBackend.class.getName();

	final Class<?> clazz1 = Class.forName(className, false, loader1);
	final Class<?> clazz2 = Class.forName(className, false, loader2);
	assertNotEquals("Test broken - the two reflectively loaded classes are equal", clazz1, clazz2);

	final Object instance1 = clazz1.getConstructor(String.class).newInstance(tmp.newFolder().toURI().toString());
	final Object instance2 = clazz2.getConstructor(String.class).newInstance(tmp.newFolder().toURI().toString());

	final String tempDir = tmp.newFolder().getAbsolutePath();

	final Method meth1 = clazz1.getDeclaredMethod("ensureRocksDBIsLoaded", String.class);
	final Method meth2 = clazz2.getDeclaredMethod("ensureRocksDBIsLoaded", String.class);
	meth1.setAccessible(true);
	meth2.setAccessible(true);

	// if all is well, these methods can both complete successfully
	meth1.invoke(instance1, tempDir);
	meth2.invoke(instance2, tempDir);
}
 

@Test
public void testTwoSeparateClassLoaders() throws Exception {
	// collect the libraries / class folders with RocksDB related code: the state backend and RocksDB itself
	final URL codePath1 = RocksDBStateBackend.class.getProtectionDomain().getCodeSource().getLocation();
	final URL codePath2 = RocksDB.class.getProtectionDomain().getCodeSource().getLocation();

	final ClassLoader parent = getClass().getClassLoader();
	final ClassLoader loader1 = FlinkUserCodeClassLoaders.childFirst(new URL[] { codePath1, codePath2 }, parent, new String[0]);
	final ClassLoader loader2 = FlinkUserCodeClassLoaders.childFirst(new URL[] { codePath1, codePath2 }, parent, new String[0]);

	final String className = RocksDBStateBackend.class.getName();

	final Class<?> clazz1 = Class.forName(className, false, loader1);
	final Class<?> clazz2 = Class.forName(className, false, loader2);
	assertNotEquals("Test broken - the two reflectively loaded classes are equal", clazz1, clazz2);

	final Object instance1 = clazz1.getConstructor(String.class).newInstance(tmp.newFolder().toURI().toString());
	final Object instance2 = clazz2.getConstructor(String.class).newInstance(tmp.newFolder().toURI().toString());

	final String tempDir = tmp.newFolder().getAbsolutePath();

	final Method meth1 = clazz1.getDeclaredMethod("ensureRocksDBIsLoaded", String.class);
	final Method meth2 = clazz2.getDeclaredMethod("ensureRocksDBIsLoaded", String.class);
	meth1.setAccessible(true);
	meth2.setAccessible(true);

	// if all is well, these methods can both complete successfully
	meth1.invoke(instance1, tempDir);
	meth2.invoke(instance2, tempDir);
}
 

public static ClassLoader buildUserCodeClassLoader(List<URL> jars, List<URL> classpaths, ClassLoader parent) {
	URL[] urls = new URL[jars.size() + classpaths.size()];
	for (int i = 0; i < jars.size(); i++) {
		urls[i] = jars.get(i);
	}
	for (int i = 0; i < classpaths.size(); i++) {
		urls[i + jars.size()] = classpaths.get(i);
	}
	return FlinkUserCodeClassLoaders.parentFirst(urls, parent);
}
 

public TaskManagerConfiguration(
		int numberSlots,
		String[] tmpDirectories,
		Time timeout,
		@Nullable Time maxRegistrationDuration,
		Time initialRegistrationPause,
		Time maxRegistrationPause,
		Time refusedRegistrationPause,
		Configuration configuration,
		boolean exitJvmOnOutOfMemory,
		FlinkUserCodeClassLoaders.ResolveOrder classLoaderResolveOrder,
		String[] alwaysParentFirstLoaderPatterns,
		@Nullable String taskManagerLogPath,
		@Nullable String taskManagerStdoutPath,
		RetryingRegistrationConfiguration retryingRegistrationConfiguration) {

	this.numberSlots = numberSlots;
	this.tmpDirectories = Preconditions.checkNotNull(tmpDirectories);
	this.timeout = Preconditions.checkNotNull(timeout);
	this.maxRegistrationDuration = maxRegistrationDuration;
	this.initialRegistrationPause = Preconditions.checkNotNull(initialRegistrationPause);
	this.maxRegistrationPause = Preconditions.checkNotNull(maxRegistrationPause);
	this.refusedRegistrationPause = Preconditions.checkNotNull(refusedRegistrationPause);
	this.configuration = new UnmodifiableConfiguration(Preconditions.checkNotNull(configuration));
	this.exitJvmOnOutOfMemory = exitJvmOnOutOfMemory;
	this.classLoaderResolveOrder = classLoaderResolveOrder;
	this.alwaysParentFirstLoaderPatterns = alwaysParentFirstLoaderPatterns;
	this.taskManagerLogPath = taskManagerLogPath;
	this.taskManagerStdoutPath = taskManagerStdoutPath;
	this.retryingRegistrationConfiguration = retryingRegistrationConfiguration;
}
 

@Test
public void testExecutionFailsInBlobsMissing() throws Exception {
	final PermanentBlobKey missingKey = new PermanentBlobKey();

	final Configuration config = new Configuration();
	config.setString(BlobServerOptions.STORAGE_DIRECTORY,
		TEMPORARY_FOLDER.newFolder().getAbsolutePath());
	config.setLong(BlobServerOptions.CLEANUP_INTERVAL, 1L);

	final BlobServer blobServer = new BlobServer(config, new VoidBlobStore());
	blobServer.start();
	InetSocketAddress serverAddress = new InetSocketAddress("localhost", blobServer.getPort());
	final PermanentBlobCache permanentBlobCache = new PermanentBlobCache(config, new VoidBlobStore(), serverAddress);

	final BlobLibraryCacheManager libraryCacheManager =
		new BlobLibraryCacheManager(
			permanentBlobCache,
			FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST,
			new String[0]);

	final Task task = new TaskBuilder()
		.setRequiredJarFileBlobKeys(Collections.singletonList(missingKey))
		.setLibraryCacheManager(libraryCacheManager)
		.build();

	// task should be new and perfect
	assertEquals(ExecutionState.CREATED, task.getExecutionState());
	assertFalse(task.isCanceledOrFailed());
	assertNull(task.getFailureCause());

	// should fail
	task.run();

	// verify final state
	assertEquals(ExecutionState.FAILED, task.getExecutionState());
	assertTrue(task.isCanceledOrFailed());
	assertNotNull(task.getFailureCause());
	assertNotNull(task.getFailureCause().getMessage());
	assertTrue(task.getFailureCause().getMessage().contains("Failed to fetch BLOB"));

	assertNull(task.getInvokable());
}
 

/**
 * Utility method to extract TaskManager config parameters from the configuration and to
 * sanity check them.
 *
 * @param configuration The configuration.
 * @param resourceID resource ID of the task manager
 * @param externalAddress identifying the IP address under which the TaskManager will be accessible
 * @param localCommunicationOnly True if only local communication is possible.
 *                               Use only in cases where only one task manager runs.
 *
 * @return configuration of task manager services used to create them
 */
public static TaskManagerServicesConfiguration fromConfiguration(
		Configuration configuration,
		ResourceID resourceID,
		String externalAddress,
		boolean localCommunicationOnly,
		TaskExecutorResourceSpec taskExecutorResourceSpec) throws Exception {
	final String[] tmpDirs = ConfigurationUtils.parseTempDirectories(configuration);
	String[] localStateRootDir = ConfigurationUtils.parseLocalStateDirectories(configuration);
	if (localStateRootDir.length == 0) {
		// default to temp dirs.
		localStateRootDir = tmpDirs;
	}

	boolean localRecoveryMode = configuration.getBoolean(CheckpointingOptions.LOCAL_RECOVERY);

	final QueryableStateConfiguration queryableStateConfig = QueryableStateConfiguration.fromConfiguration(configuration);

	long timerServiceShutdownTimeout = AkkaUtils.getTimeout(configuration).toMillis();

	final RetryingRegistrationConfiguration retryingRegistrationConfiguration = RetryingRegistrationConfiguration.fromConfiguration(configuration);

	final int externalDataPort = configuration.getInteger(NettyShuffleEnvironmentOptions.DATA_PORT);

	String bindAddr = configuration.getString(TaskManagerOptions.BIND_HOST, NetUtils.getWildcardIPAddress());
	InetAddress bindAddress = InetAddress.getByName(bindAddr);

	final String classLoaderResolveOrder =
		configuration.getString(CoreOptions.CLASSLOADER_RESOLVE_ORDER);

	final String[] alwaysParentFirstLoaderPatterns = CoreOptions.getParentFirstLoaderPatterns(configuration);

	final int numIoThreads = ClusterEntrypointUtils.getPoolSize(configuration);

	return new TaskManagerServicesConfiguration(
		configuration,
		resourceID,
		externalAddress,
		bindAddress,
		externalDataPort,
		localCommunicationOnly,
		tmpDirs,
		localStateRootDir,
		localRecoveryMode,
		queryableStateConfig,
		ConfigurationParserUtils.getSlot(configuration),
		ConfigurationParserUtils.getPageSize(configuration),
		taskExecutorResourceSpec,
		timerServiceShutdownTimeout,
		retryingRegistrationConfiguration,
		ConfigurationUtils.getSystemResourceMetricsProbingInterval(configuration),
		FlinkUserCodeClassLoaders.ResolveOrder.fromString(classLoaderResolveOrder),
		alwaysParentFirstLoaderPatterns,
		numIoThreads);
}
 

public FlinkUserCodeClassLoaders.ResolveOrder getClassLoaderResolveOrder() {
	return classLoaderResolveOrder;
}
 

private TaskManagerServicesConfiguration(
		Configuration configuration,
		ResourceID resourceID,
		String externalAddress,
		InetAddress bindAddress,
		int externalDataPort,
		boolean localCommunicationOnly,
		String[] tmpDirPaths,
		String[] localRecoveryStateRootDirectories,
		boolean localRecoveryEnabled,
		@Nullable QueryableStateConfiguration queryableStateConfig,
		int numberOfSlots,
		int pageSize,
		TaskExecutorResourceSpec taskExecutorResourceSpec,
		long timerServiceShutdownTimeout,
		RetryingRegistrationConfiguration retryingRegistrationConfiguration,
		Optional<Time> systemResourceMetricsProbingInterval,
		FlinkUserCodeClassLoaders.ResolveOrder classLoaderResolveOrder,
		String[] alwaysParentFirstLoaderPatterns,
		int numIoThreads) {
	this.configuration = checkNotNull(configuration);
	this.resourceID = checkNotNull(resourceID);

	this.externalAddress = checkNotNull(externalAddress);
	this.bindAddress = checkNotNull(bindAddress);
	this.externalDataPort = externalDataPort;
	this.localCommunicationOnly = localCommunicationOnly;
	this.tmpDirPaths = checkNotNull(tmpDirPaths);
	this.localRecoveryStateRootDirectories = checkNotNull(localRecoveryStateRootDirectories);
	this.localRecoveryEnabled = checkNotNull(localRecoveryEnabled);
	this.queryableStateConfig = queryableStateConfig;
	this.numberOfSlots = checkNotNull(numberOfSlots);

	this.pageSize = pageSize;

	this.taskExecutorResourceSpec = taskExecutorResourceSpec;
	this.classLoaderResolveOrder = classLoaderResolveOrder;
	this.alwaysParentFirstLoaderPatterns = alwaysParentFirstLoaderPatterns;
	this.numIoThreads = numIoThreads;

	checkArgument(timerServiceShutdownTimeout >= 0L, "The timer " +
		"service shutdown timeout must be greater or equal to 0.");
	this.timerServiceShutdownTimeout = timerServiceShutdownTimeout;
	this.retryingRegistrationConfiguration = checkNotNull(retryingRegistrationConfiguration);

	this.systemResourceMetricsProbingInterval = checkNotNull(systemResourceMetricsProbingInterval);
}
 

public static JobManagerSharedServices fromConfiguration(
		Configuration config,
		BlobServer blobServer,
		FatalErrorHandler fatalErrorHandler) {

	checkNotNull(config);
	checkNotNull(blobServer);

	final String classLoaderResolveOrder =
		config.getString(CoreOptions.CLASSLOADER_RESOLVE_ORDER);

	final String[] alwaysParentFirstLoaderPatterns = CoreOptions.getParentFirstLoaderPatterns(config);

	final boolean failOnJvmMetaspaceOomError = config.getBoolean(CoreOptions.FAIL_ON_USER_CLASS_LOADING_METASPACE_OOM);
	final BlobLibraryCacheManager libraryCacheManager =
		new BlobLibraryCacheManager(
			blobServer,
			BlobLibraryCacheManager.defaultClassLoaderFactory(
				FlinkUserCodeClassLoaders.ResolveOrder.fromString(classLoaderResolveOrder),
				alwaysParentFirstLoaderPatterns,
				failOnJvmMetaspaceOomError ? fatalErrorHandler : null));

	final Duration akkaTimeout;
	try {
		akkaTimeout = AkkaUtils.getTimeout(config);
	} catch (NumberFormatException e) {
		throw new IllegalConfigurationException(AkkaUtils.formatDurationParsingErrorMessage());
	}

	final ScheduledExecutorService futureExecutor = Executors.newScheduledThreadPool(
			Hardware.getNumberCPUCores(),
			new ExecutorThreadFactory("jobmanager-future"));

	final int numSamples = config.getInteger(WebOptions.BACKPRESSURE_NUM_SAMPLES);
	final long delayBetweenSamples = config.getInteger(WebOptions.BACKPRESSURE_DELAY);
	final BackPressureRequestCoordinator coordinator = new BackPressureRequestCoordinator(
		futureExecutor,
		akkaTimeout.toMillis() + numSamples * delayBetweenSamples);

	final int cleanUpInterval = config.getInteger(WebOptions.BACKPRESSURE_CLEANUP_INTERVAL);
	final BackPressureStatsTrackerImpl backPressureStatsTracker = new BackPressureStatsTrackerImpl(
		coordinator,
		cleanUpInterval,
		config.getInteger(WebOptions.BACKPRESSURE_REFRESH_INTERVAL));

	futureExecutor.scheduleWithFixedDelay(
		backPressureStatsTracker::cleanUpOperatorStatsCache,
		cleanUpInterval,
		cleanUpInterval,
		TimeUnit.MILLISECONDS);

	return new JobManagerSharedServices(
		futureExecutor,
		libraryCacheManager,
		coordinator,
		backPressureStatsTracker,
		blobServer);
}