org.apache.flink.testutils.ClassLoaderUtils Java Examples

The following examples show how to use org.apache.flink.testutils.ClassLoaderUtils. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: KryoSerializerSnapshotTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * This method returns the bytes of a serialized {@link KryoSerializerSnapshot}, that contains a Kryo registration
 * of a class that does not exists in the current classpath.
 */
private static byte[] unLoadableSnapshotBytes() throws IOException {
	final ClassLoader originalClassLoader = Thread.currentThread().getContextClassLoader();

	final ClassLoaderUtils.ObjectAndClassLoader<Serializable> outsideClassLoading = ClassLoaderUtils.createSerializableObjectFromNewClassLoader();

	try {
		Thread.currentThread().setContextClassLoader(outsideClassLoading.getClassLoader());

		ExecutionConfig conf = new ExecutionConfig();
		conf.registerKryoType(outsideClassLoading.getObject().getClass());

		KryoSerializer<Animal> previousSerializer = new KryoSerializer<>(Animal.class, conf);
		TypeSerializerSnapshot<Animal> previousSnapshot = previousSerializer.snapshotConfiguration();

		DataOutputSerializer out = new DataOutputSerializer(4096);
		TypeSerializerSnapshot.writeVersionedSnapshot(out, previousSnapshot);
		return out.getCopyOfBuffer();
	}
	finally {
		Thread.currentThread().setContextClassLoader(originalClassLoader);
	}
}
Example #2
Source File: HadoopFreeFsFactoryTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * This test validates that the factory can be instantiated and configured even
 * when Hadoop classes are missing from the classpath.
 */
@Test
public void testHadoopFactoryInstantiationWithoutHadoop() throws Exception {
	// we do reflection magic here to instantiate the test in another class
	// loader, to make sure no hadoop classes are in the classpath

	final String testClassName = "org.apache.flink.runtime.fs.hdfs.HadoopFreeTests";

	final URL[] urls = ClassLoaderUtils.getClasspathURLs();

	ClassLoader parent = getClass().getClassLoader();
	ClassLoader hadoopFreeClassLoader = new HadoopFreeClassLoader(urls, parent);
	Class<?> testClass = Class.forName(testClassName, false, hadoopFreeClassLoader);
	Method m = testClass.getDeclaredMethod("test");

	try {
		m.invoke(null);
	}
	catch (InvocationTargetException e) {
		ExceptionUtils.rethrowException(e.getTargetException(), "exception in method");
	}
}
Example #3
Source File: PackagedProgramUtilsPipelineTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
private List<URL> getClassUrls(String className) throws IOException {
	URLClassLoader urlClassLoader = ClassLoaderUtils.compileAndLoadJava(
		temporaryFolder.newFolder(),
		className + ".java",
		"import com.esotericsoftware.kryo.Kryo;\n" +
			"import com.esotericsoftware.kryo.Serializer;\n" +
			"import com.esotericsoftware.kryo.io.Input;\n" +
			"import com.esotericsoftware.kryo.io.Output;\n"
			+ "public class " + className + " extends Serializer {\n" +
			"\t@Override\n" +
			"\tpublic void write(\n" +
			"\t\tKryo kryo,\n" +
			"\t\tOutput output,\n" +
			"\t\tObject object) {\n" +
			"\t}\n" +
			"\n" +
			"\t@Override\n" +
			"\tpublic Object read(Kryo kryo, Input input, Class type) {\n" +
			"\t\treturn null;\n" +
			"\t}\n" +
			"}");
	return Arrays.asList(urlClassLoader.getURLs());
}
Example #4
Source File: EnumSerializerUpgradeTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
private static TypeSerializerSchemaCompatibility checkCompatibility(String enumSourceA, String enumSourceB)
	throws IOException, ClassNotFoundException {

	ClassLoader classLoader = ClassLoaderUtils.compileAndLoadJava(
		temporaryFolder.newFolder(), ENUM_NAME + ".java", enumSourceA);

	EnumSerializer enumSerializer = new EnumSerializer(classLoader.loadClass(ENUM_NAME));

	TypeSerializerSnapshot snapshot = enumSerializer.snapshotConfiguration();
	byte[] snapshotBytes;
	try (
		ByteArrayOutputStream outBuffer = new ByteArrayOutputStream();
		DataOutputViewStreamWrapper outputViewStreamWrapper = new DataOutputViewStreamWrapper(outBuffer)) {

		TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot(
			outputViewStreamWrapper, snapshot, enumSerializer);
		snapshotBytes = outBuffer.toByteArray();
	}

	ClassLoader classLoader2 = ClassLoaderUtils.compileAndLoadJava(
		temporaryFolder.newFolder(), ENUM_NAME + ".java", enumSourceB);

	TypeSerializerSnapshot restoredSnapshot;
	try (
		ByteArrayInputStream inBuffer = new ByteArrayInputStream(snapshotBytes);
		DataInputViewStreamWrapper inputViewStreamWrapper = new DataInputViewStreamWrapper(inBuffer)) {

		restoredSnapshot = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot(
			inputViewStreamWrapper, classLoader2, enumSerializer);
	}

	EnumSerializer enumSerializer2 = new EnumSerializer(classLoader2.loadClass(ENUM_NAME));
	return restoredSnapshot.resolveSchemaCompatibility(enumSerializer2);
}
Example #5
Source File: ClassLoaderTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Test
public void testMessageDecodingWithUnavailableClass() throws Exception {
	final ClassLoader systemClassLoader = getClass().getClassLoader();

	final String className = "UserClass";
	final URLClassLoader userClassLoader = ClassLoaderUtils.compileAndLoadJava(
		temporaryFolder.newFolder(),
		className + ".java",
		"import java.io.Serializable;\n"
			+ "public class " + className + " implements Serializable {}");

	RemoteRpcInvocation method = new RemoteRpcInvocation(
		"test",
		new Class<?>[] {
			int.class,
			Class.forName(className, false, userClassLoader)},
		new Object[] {
			1,
			Class.forName(className, false, userClassLoader).newInstance()});

	SerializedValue<RemoteRpcInvocation> serializedMethod = new SerializedValue<>(method);

	expectedException.expect(ClassNotFoundException.class);
	expectedException.expect(
		allOf(
			isA(ClassNotFoundException.class),
			hasProperty("suppressed",
				hasItemInArray(
					allOf(
						isA(ClassNotFoundException.class),
						hasProperty("message",
							containsString("Could not deserialize 1th parameter type of method test(int, ...).")))))));

	RemoteRpcInvocation deserializedMethod = serializedMethod.deserializeValue(systemClassLoader);
	deserializedMethod.getMethodName();

	userClassLoader.close();
}
Example #6
Source File: EnumSerializerUpgradeTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
private static TypeSerializerSchemaCompatibility checkCompatibility(String enumSourceA, String enumSourceB)
	throws IOException, ClassNotFoundException {

	ClassLoader classLoader = ClassLoaderUtils.compileAndLoadJava(
		temporaryFolder.newFolder(), ENUM_NAME + ".java", enumSourceA);

	EnumSerializer enumSerializer = new EnumSerializer(classLoader.loadClass(ENUM_NAME));

	TypeSerializerSnapshot snapshot = enumSerializer.snapshotConfiguration();
	byte[] snapshotBytes;
	try (
		ByteArrayOutputStream outBuffer = new ByteArrayOutputStream();
		DataOutputViewStreamWrapper outputViewStreamWrapper = new DataOutputViewStreamWrapper(outBuffer)) {

		TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot(
			outputViewStreamWrapper, snapshot, enumSerializer);
		snapshotBytes = outBuffer.toByteArray();
	}

	ClassLoader classLoader2 = ClassLoaderUtils.compileAndLoadJava(
		temporaryFolder.newFolder(), ENUM_NAME + ".java", enumSourceB);

	TypeSerializerSnapshot restoredSnapshot;
	try (
		ByteArrayInputStream inBuffer = new ByteArrayInputStream(snapshotBytes);
		DataInputViewStreamWrapper inputViewStreamWrapper = new DataInputViewStreamWrapper(inBuffer)) {

		restoredSnapshot = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot(
			inputViewStreamWrapper, classLoader2, enumSerializer);
	}

	EnumSerializer enumSerializer2 = new EnumSerializer(classLoader2.loadClass(ENUM_NAME));
	return restoredSnapshot.resolveSchemaCompatibility(enumSerializer2);
}
Example #7
Source File: ClassLoaderTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testMessageDecodingWithUnavailableClass() throws Exception {
	final ClassLoader systemClassLoader = getClass().getClassLoader();

	final String className = "UserClass";
	final URLClassLoader userClassLoader = ClassLoaderUtils.compileAndLoadJava(
		temporaryFolder.newFolder(),
		className + ".java",
		"import java.io.Serializable;\n"
			+ "public class " + className + " implements Serializable {}");

	RemoteRpcInvocation method = new RemoteRpcInvocation(
		"test",
		new Class<?>[] {
			int.class,
			Class.forName(className, false, userClassLoader)},
		new Object[] {
			1,
			Class.forName(className, false, userClassLoader).newInstance()});

	SerializedValue<RemoteRpcInvocation> serializedMethod = new SerializedValue<>(method);

	expectedException.expect(ClassNotFoundException.class);
	expectedException.expect(
		allOf(
			isA(ClassNotFoundException.class),
			hasProperty("suppressed",
				hasItemInArray(
					allOf(
						isA(ClassNotFoundException.class),
						hasProperty("message",
							containsString("Could not deserialize 1th parameter type of method test(int, ...).")))))));

	RemoteRpcInvocation deserializedMethod = serializedMethod.deserializeValue(systemClassLoader);
	deserializedMethod.getMethodName();

	userClassLoader.close();
}
Example #8
Source File: ExceptionUtilsITCases.java    From flink with Apache License 2.0 5 votes vote down vote up 
private static Class<?> loadDummyClass(int index, String folderToSaveSource) throws ClassNotFoundException, IOException {
	String className = "DummyClass" + index;
	String sourcePattern = "public class %s { @Override public String toString() { return \"%s\"; } }";
	ClassLoaderBuilder classLoaderBuilder = ClassLoaderUtils.withRoot(new File(folderToSaveSource));
	classLoaderBuilder.addClass(className, String.format(sourcePattern, className, "dummy"));
	ClassLoader classLoader = classLoaderBuilder.build();
	return Class.forName(className, true, classLoader);
}
Example #9
Source File: EnumSerializerCompatibilityTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
private static TypeSerializerSchemaCompatibility checkCompatibility(String enumSourceA, String enumSourceB)
	throws IOException, ClassNotFoundException {

	ClassLoader classLoader = ClassLoaderUtils.compileAndLoadJava(
		temporaryFolder.newFolder(), ENUM_NAME + ".java", enumSourceA);

	EnumSerializer enumSerializer = new EnumSerializer(classLoader.loadClass(ENUM_NAME));

	TypeSerializerSnapshot snapshot = enumSerializer.snapshotConfiguration();
	byte[] snapshotBytes;
	try (
		ByteArrayOutputStream outBuffer = new ByteArrayOutputStream();
		DataOutputViewStreamWrapper outputViewStreamWrapper = new DataOutputViewStreamWrapper(outBuffer)) {

		TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot(
			outputViewStreamWrapper, snapshot, enumSerializer);
		snapshotBytes = outBuffer.toByteArray();
	}

	ClassLoader classLoader2 = ClassLoaderUtils.compileAndLoadJava(
		temporaryFolder.newFolder(), ENUM_NAME + ".java", enumSourceB);

	TypeSerializerSnapshot restoredSnapshot;
	try (
		ByteArrayInputStream inBuffer = new ByteArrayInputStream(snapshotBytes);
		DataInputViewStreamWrapper inputViewStreamWrapper = new DataInputViewStreamWrapper(inBuffer)) {

		restoredSnapshot = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot(
			inputViewStreamWrapper, classLoader2, enumSerializer);
	}

	EnumSerializer enumSerializer2 = new EnumSerializer(classLoader2.loadClass(ENUM_NAME));
	return restoredSnapshot.resolveSchemaCompatibility(enumSerializer2);
}
Example #10
Source File: FlinkUserCodeClassLoadersTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testMessageDecodingWithUnavailableClass() throws Exception {
	final ClassLoader systemClassLoader = getClass().getClassLoader();

	final String className = "UserClass";
	final URLClassLoader userClassLoader = ClassLoaderUtils.compileAndLoadJava(
		temporaryFolder.newFolder(),
		className + ".java",
		"import java.io.Serializable;\n"
			+ "public class " + className + " implements Serializable {}");

	RemoteRpcInvocation method = new RemoteRpcInvocation(
		"test",
		new Class<?>[] {
			int.class,
			Class.forName(className, false, userClassLoader)},
		new Object[] {
			1,
			Class.forName(className, false, userClassLoader).newInstance()});

	SerializedValue<RemoteRpcInvocation> serializedMethod = new SerializedValue<>(method);

	expectedException.expect(ClassNotFoundException.class);
	expectedException.expect(
		allOf(
			isA(ClassNotFoundException.class),
			hasProperty("suppressed",
				hasItemInArray(
					allOf(
						isA(ClassNotFoundException.class),
						hasProperty("message",
							containsString("Could not deserialize 1th parameter type of method test(int, ...).")))))));

	RemoteRpcInvocation deserializedMethod = serializedMethod.deserializeValue(systemClassLoader);
	deserializedMethod.getMethodName();

	userClassLoader.close();
}
Example #11
Source File: InstantiationUtilTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
private URLClassLoader createClassLoader(String interfaceName, String proxyName) throws IOException {
	return ClassLoaderUtils.withRoot(temporaryFolder.newFolder())
		.addClass(interfaceName, String.format("interface %s { void test();}", interfaceName))
		.addClass(proxyName, createProxyDefinition(proxyName))
		.build();
}
Example #12
Source File: InstantiationUtilTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
private URLClassLoader createClassLoader(String interfaceName, String proxyName) throws IOException {
	return ClassLoaderUtils.withRoot(temporaryFolder.newFolder())
		.addClass(interfaceName, String.format("interface %s { void test();}", interfaceName))
		.addClass(proxyName, createProxyDefinition(proxyName))
		.build();
}
Example #13
Source File: InstantiationUtilTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
private URLClassLoader createClassLoader(String interfaceName, String proxyName) throws IOException {
	return ClassLoaderUtils.withRoot(temporaryFolder.newFolder())
		.addClass(interfaceName, String.format("interface %s { void test();}", interfaceName))
		.addClass(proxyName, createProxyDefinition(proxyName))
		.build();
}
Example #14
Source File: MapRNotInClassPathTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
MapRFreeClassLoader(ClassLoader parent) {
	super(ClassLoaderUtils.getClasspathURLs(), null);
	properParent = parent;
}
Example #15
Source File: CheckpointSettingsSerializableTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Test
public void testDeserializationOfUserCodeWithUserClassLoader() throws Exception {
	final ClassLoaderUtils.ObjectAndClassLoader<Serializable> outsideClassLoading = ClassLoaderUtils.createSerializableObjectFromNewClassLoader();
	final ClassLoader classLoader = outsideClassLoading.getClassLoader();
	final Serializable outOfClassPath = outsideClassLoading.getObject();

	final MasterTriggerRestoreHook.Factory[] hooks = {
			new TestFactory(outOfClassPath) };
	final SerializedValue<MasterTriggerRestoreHook.Factory[]> serHooks = new SerializedValue<>(hooks);

	final JobCheckpointingSettings checkpointingSettings = new JobCheckpointingSettings(
			Collections.<JobVertexID>emptyList(),
			Collections.<JobVertexID>emptyList(),
			Collections.<JobVertexID>emptyList(),
			new CheckpointCoordinatorConfiguration(
				1000L,
				10000L,
				0L,
				1,
				CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION,
				true,
				false,
				false,
				0),
			new SerializedValue<StateBackend>(new CustomStateBackend(outOfClassPath)),
			serHooks);

	final JobGraph jobGraph = new JobGraph(new JobID(), "test job");
	jobGraph.setSnapshotSettings(checkpointingSettings);

	// to serialize/deserialize the job graph to see if the behavior is correct under
	// distributed execution
	final JobGraph copy = CommonTestUtils.createCopySerializable(jobGraph);

	final Time timeout = Time.seconds(10L);
	final ExecutionGraph eg = ExecutionGraphBuilder.buildGraph(
		null,
		copy,
		new Configuration(),
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		mock(SlotProvider.class),
		classLoader,
		new StandaloneCheckpointRecoveryFactory(),
		timeout,
		new NoRestartStrategy(),
		new UnregisteredMetricsGroup(),
		VoidBlobWriter.getInstance(),
		timeout,
		log,
		NettyShuffleMaster.INSTANCE,
		NoOpJobMasterPartitionTracker.INSTANCE);

	assertEquals(1, eg.getCheckpointCoordinator().getNumberOfRegisteredMasterHooks());
	assertTrue(jobGraph.getCheckpointingSettings().getDefaultStateBackend().deserializeValue(classLoader) instanceof CustomStateBackend);
}