Java Code Examples for org.apache.flink.client.program.PackagedProgram#invokeInteractiveModeForExecution()

@Test
public void testExternalProgram() throws Exception {

	String jarFile = JAR_FILE;
	try {
		JobWithJars.checkJarFile(new File(jarFile).getAbsoluteFile().toURI().toURL());
	} catch (IOException e) {
		jarFile = "target/".concat(jarFile);
	}

	TestEnvironment.setAsContext(
		MINI_CLUSTER,
		PARALLELISM,
		Collections.singleton(new Path(jarFile)),
		Collections.emptyList());

	String testData = getClass().getResource(TEST_DATA_FILE).toString();

	PackagedProgram program = new PackagedProgram(new File(jarFile), new String[]{testData});

	program.invokeInteractiveModeForExecution();
}
 

@Test
public void testKMeansJobWithCustomClassLoader() throws ProgramInvocationException {
	PackagedProgram kMeansProg = PackagedProgram.newBuilder()
		.setJarFile(new File(KMEANS_JAR_PATH))
		.setArguments(new String[] {
			KMeansData.DATAPOINTS,
			KMeansData.INITIAL_CENTERS,
			"25"})
		.build();

	TestEnvironment.setAsContext(
		miniClusterResource.getMiniCluster(),
		parallelism,
		Collections.singleton(new Path(KMEANS_JAR_PATH)),
		Collections.emptyList());

	kMeansProg.invokeInteractiveModeForExecution();
}
 

@Test
public void testExternalProgram() throws Exception {

	String jarFile = JAR_FILE;
	try {
		JobWithJars.checkJarFile(new File(jarFile).getAbsoluteFile().toURI().toURL());
	} catch (IOException e) {
		jarFile = "target/".concat(jarFile);
	}

	TestEnvironment.setAsContext(
		MINI_CLUSTER,
		PARALLELISM,
		Collections.singleton(new Path(jarFile)),
		Collections.emptyList());

	String testData = getClass().getResource(TEST_DATA_FILE).toString();

	PackagedProgram program = new PackagedProgram(new File(jarFile), new String[]{testData});

	program.invokeInteractiveModeForExecution();
}
 

@Test
public void testCheckpointingCustomKvStateJobWithCustomClassLoader() throws IOException, ProgramInvocationException {
	File checkpointDir = FOLDER.newFolder();
	File outputDir = FOLDER.newFolder();

	final PackagedProgram program = new PackagedProgram(
		new File(CHECKPOINTING_CUSTOM_KV_STATE_JAR_PATH),
		new String[] {
			checkpointDir.toURI().toString(),
			outputDir.toURI().toString()
		});

	TestStreamEnvironment.setAsContext(
		miniClusterResource.getMiniCluster(),
		parallelism,
		Collections.singleton(new Path(CHECKPOINTING_CUSTOM_KV_STATE_JAR_PATH)),
		Collections.<URL>emptyList());

	expectedException.expectCause(
		Matchers.<Throwable>hasProperty("cause", isA(SuccessException.class)));

	program.invokeInteractiveModeForExecution();
}
 

@Test
public void testKMeansJobWithCustomClassLoader() throws IOException, ProgramInvocationException {
	PackagedProgram kMeansProg = new PackagedProgram(
		new File(KMEANS_JAR_PATH),
		new String[] {
			KMeansData.DATAPOINTS,
			KMeansData.INITIAL_CENTERS,
			"25"
		});

	TestEnvironment.setAsContext(
		miniClusterResource.getMiniCluster(),
		parallelism,
		Collections.singleton(new Path(KMEANS_JAR_PATH)),
		Collections.<URL>emptyList());

	kMeansProg.invokeInteractiveModeForExecution();
}
 

@Test
public void testCheckpointingCustomKvStateJobWithCustomClassLoader() throws IOException, ProgramInvocationException {
	File checkpointDir = FOLDER.newFolder();
	File outputDir = FOLDER.newFolder();

	final PackagedProgram program = PackagedProgram.newBuilder()
		.setJarFile(new File(CHECKPOINTING_CUSTOM_KV_STATE_JAR_PATH))
		.setArguments(new String[] { checkpointDir.toURI().toString(), outputDir.toURI().toString()})
		.build();

	TestStreamEnvironment.setAsContext(
		miniClusterResource.getMiniCluster(),
		parallelism,
		Collections.singleton(new Path(CHECKPOINTING_CUSTOM_KV_STATE_JAR_PATH)),
		Collections.emptyList());

	try {
		program.invokeInteractiveModeForExecution();
		fail("exception should happen");
	} catch (ProgramInvocationException e) {
		assertTrue(ExceptionUtils.findThrowable(e, SuccessException.class).isPresent());
	}
}
 

@Test
public void testStreamingClassloaderJobWithCustomClassLoader() throws IOException, ProgramInvocationException {
	// regular streaming job
	PackagedProgram streamingProg = new PackagedProgram(new File(STREAMING_PROG_JAR_FILE));

	TestStreamEnvironment.setAsContext(
		miniClusterResource.getMiniCluster(),
		parallelism,
		Collections.singleton(new Path(STREAMING_PROG_JAR_FILE)),
		Collections.<URL>emptyList());

	streamingProg.invokeInteractiveModeForExecution();
}
 

@Test
public void testUserCodeTypeJobWithCustomClassLoader() throws IOException, ProgramInvocationException {
	PackagedProgram userCodeTypeProg = new PackagedProgram(new File(USERCODETYPE_JAR_PATH));

	TestEnvironment.setAsContext(
		miniClusterResource.getMiniCluster(),
		parallelism,
		Collections.singleton(new Path(USERCODETYPE_JAR_PATH)),
		Collections.<URL>emptyList());

	userCodeTypeProg.invokeInteractiveModeForExecution();
}
 

@Test
public void testStreamingCustomSplitJobWithCustomClassLoader() throws IOException, ProgramInvocationException {
	PackagedProgram streamingInputSplitTestProg = new PackagedProgram(new File(STREAMING_INPUT_SPLITS_PROG_JAR_FILE));

	TestStreamEnvironment.setAsContext(
		miniClusterResource.getMiniCluster(),
		parallelism,
		Collections.singleton(new Path(STREAMING_INPUT_SPLITS_PROG_JAR_FILE)),
		Collections.<URL>emptyList());

	streamingInputSplitTestProg.invokeInteractiveModeForExecution();
}
 

@Test
public void testCustomSplitJobWithCustomClassLoaderJar() throws IOException, ProgramInvocationException {

	PackagedProgram inputSplitTestProg = new PackagedProgram(new File(INPUT_SPLITS_PROG_JAR_FILE));

	TestEnvironment.setAsContext(
		miniClusterResource.getMiniCluster(),
		parallelism,
		Collections.singleton(new Path(INPUT_SPLITS_PROG_JAR_FILE)),
		Collections.<URL>emptyList());

	inputSplitTestProg.invokeInteractiveModeForExecution();
}
 

@Test
public void testStreamingCustomSplitJobWithCustomClassLoader() throws IOException, ProgramInvocationException {
	PackagedProgram streamingInputSplitTestProg = new PackagedProgram(new File(STREAMING_INPUT_SPLITS_PROG_JAR_FILE));

	TestStreamEnvironment.setAsContext(
		miniClusterResource.getMiniCluster(),
		parallelism,
		Collections.singleton(new Path(STREAMING_INPUT_SPLITS_PROG_JAR_FILE)),
		Collections.<URL>emptyList());

	streamingInputSplitTestProg.invokeInteractiveModeForExecution();
}
 

@Test
public void testStreamingCustomSplitJobWithCustomClassLoader() throws ProgramInvocationException {
	PackagedProgram streamingInputSplitTestProg = PackagedProgram.newBuilder()
		.setJarFile(new File(STREAMING_INPUT_SPLITS_PROG_JAR_FILE))
		.build();

	TestStreamEnvironment.setAsContext(
		miniClusterResource.getMiniCluster(),
		parallelism,
		Collections.singleton(new Path(STREAMING_INPUT_SPLITS_PROG_JAR_FILE)),
		Collections.emptyList());

	streamingInputSplitTestProg.invokeInteractiveModeForExecution();
}
 

@Test
public void testStreamingClassloaderJobWithCustomClassLoader() throws IOException, ProgramInvocationException {
	// regular streaming job
	PackagedProgram streamingProg = new PackagedProgram(new File(STREAMING_PROG_JAR_FILE));

	TestStreamEnvironment.setAsContext(
		miniClusterResource.getMiniCluster(),
		parallelism,
		Collections.singleton(new Path(STREAMING_PROG_JAR_FILE)),
		Collections.<URL>emptyList());

	streamingProg.invokeInteractiveModeForExecution();
}
 

@Test
public void testCheckpointedStreamingClassloaderJobWithCustomClassLoader() throws IOException, ProgramInvocationException {
	// checkpointed streaming job with custom classes for the checkpoint (FLINK-2543)
	// the test also ensures that user specific exceptions are serializable between JobManager <--> JobClient.
	PackagedProgram streamingCheckpointedProg = new PackagedProgram(new File(STREAMING_CHECKPOINTED_PROG_JAR_FILE));

	TestStreamEnvironment.setAsContext(
		miniClusterResource.getMiniCluster(),
		parallelism,
		Collections.singleton(new Path(STREAMING_CHECKPOINTED_PROG_JAR_FILE)),
		Collections.<URL>emptyList());

	try {
		streamingCheckpointedProg.invokeInteractiveModeForExecution();
	} catch (Exception e) {
		// Program should terminate with a 'SuccessException':
		// the exception class is contained in the user-jar, but is not present on the maven classpath
		// the deserialization of the exception should thus fail here
		try {
			Optional<Throwable> exception = ExceptionUtils.findThrowable(e,
				candidate -> candidate.getClass().getCanonicalName().equals("org.apache.flink.test.classloading.jar.CheckpointedStreamingProgram.SuccessException"));

			// if we reach this point we either failed due to another exception,
			// or the deserialization of the user-exception did not fail
			if (!exception.isPresent()) {
				throw e;
			} else {
				Assert.fail("Deserialization of user exception should have failed.");
			}
		} catch (NoClassDefFoundError expected) {
			// expected
		}
	}
}
 

@Test
public void testUserCodeTypeJobWithCustomClassLoader() throws ProgramInvocationException {
	PackagedProgram userCodeTypeProg = PackagedProgram.newBuilder()
		.setJarFile(new File(USERCODETYPE_JAR_PATH))
		.build();

	TestEnvironment.setAsContext(
		miniClusterResource.getMiniCluster(),
		parallelism,
		Collections.singleton(new Path(USERCODETYPE_JAR_PATH)),
		Collections.emptyList());

	userCodeTypeProg.invokeInteractiveModeForExecution();
}
 

@Test
public void testCustomSplitJobWithCustomClassLoaderJar() throws ProgramInvocationException {

	PackagedProgram inputSplitTestProg = PackagedProgram.newBuilder()
		.setJarFile(new File(INPUT_SPLITS_PROG_JAR_FILE))
		.build();

	TestEnvironment.setAsContext(
		miniClusterResource.getMiniCluster(),
		parallelism,
		Collections.singleton(new Path(INPUT_SPLITS_PROG_JAR_FILE)),
		Collections.emptyList());

	inputSplitTestProg.invokeInteractiveModeForExecution();
}
 

@Test
public void testProgramWithParentFirstClassLoader() throws IOException, ProgramInvocationException {
	// We have two files named test-resource in src/resource (parent classloader classpath) and
	// tmp folders (child classloader classpath) respectively.
	String childResourceDirName = "child1";
	String testResourceName = "test-resource";
	File childResourceDir = FOLDER.newFolder(childResourceDirName);
	File childResource = new File(childResourceDir, testResourceName);
	assertTrue(childResource.createNewFile());

	TestStreamEnvironment.setAsContext(
		miniClusterResource.getMiniCluster(),
		parallelism,
		Collections.singleton(new Path(CLASSLOADING_POLICY_JAR_PATH)),
		Collections.emptyList());

	// parent-first classloading
	Configuration parentFirstConf = new Configuration();
	parentFirstConf.setString("classloader.resolve-order", "parent-first");

	final PackagedProgram parentFirstProgram = PackagedProgram.newBuilder()
		.setJarFile(new File(CLASSLOADING_POLICY_JAR_PATH))
		.setUserClassPaths(Collections.singletonList(childResourceDir.toURI().toURL()))
		.setConfiguration(parentFirstConf)
		.setArguments(testResourceName, "test-classes")
		.build();

	final ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
	Thread.currentThread().setContextClassLoader(parentFirstProgram.getUserCodeClassLoader());
	try {
		parentFirstProgram.invokeInteractiveModeForExecution();
	} finally {
		Thread.currentThread().setContextClassLoader(contextClassLoader);
	}
}
 

@Test
public void testCustomSplitJobWithCustomClassLoaderJar() throws IOException, ProgramInvocationException {

	PackagedProgram inputSplitTestProg = new PackagedProgram(new File(INPUT_SPLITS_PROG_JAR_FILE));

	TestEnvironment.setAsContext(
		miniClusterResource.getMiniCluster(),
		parallelism,
		Collections.singleton(new Path(INPUT_SPLITS_PROG_JAR_FILE)),
		Collections.<URL>emptyList());

	inputSplitTestProg.invokeInteractiveModeForExecution();
}
 

/**
 * Tests disposal of a savepoint, which contains custom user code KvState.
 */
@Test
public void testDisposeSavepointWithCustomKvState() throws Exception {
	ClusterClient<?> clusterClient = new MiniClusterClient(new Configuration(), miniClusterResource.getMiniCluster());

	Deadline deadline = new FiniteDuration(100, TimeUnit.SECONDS).fromNow();

	File checkpointDir = FOLDER.newFolder();
	File outputDir = FOLDER.newFolder();

	final PackagedProgram program = PackagedProgram.newBuilder()
		.setJarFile(new File(CUSTOM_KV_STATE_JAR_PATH))
		.setArguments(new String[] {
			String.valueOf(parallelism),
			checkpointDir.toURI().toString(),
			"5000",
			outputDir.toURI().toString(),
			"false" // Disable unaligned checkpoints as this test is triggering concurrent savepoints/checkpoints
		})
		.build();

	TestStreamEnvironment.setAsContext(
		miniClusterResource.getMiniCluster(),
		parallelism,
		Collections.singleton(new Path(CUSTOM_KV_STATE_JAR_PATH)),
		Collections.emptyList()
	);

	// Execute detached
	Thread invokeThread = new Thread(() -> {
		try {
			program.invokeInteractiveModeForExecution();
		} catch (ProgramInvocationException ex) {
			if (ex.getCause() == null ||
				!(ex.getCause() instanceof JobCancellationException)) {
				ex.printStackTrace();
			}
		}
	});

	LOG.info("Starting program invoke thread");
	invokeThread.start();

	// The job ID
	JobID jobId = null;

	LOG.info("Waiting for job status running.");

	// Wait for running job
	while (jobId == null && deadline.hasTimeLeft()) {

		Collection<JobStatusMessage> jobs = clusterClient.listJobs().get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS);
		for (JobStatusMessage job : jobs) {
			if (job.getJobState() == JobStatus.RUNNING) {
				jobId = job.getJobId();
				LOG.info("Job running. ID: " + jobId);
				break;
			}
		}

		// Retry if job is not available yet
		if (jobId == null) {
			Thread.sleep(100L);
		}
	}

	// Trigger savepoint
	String savepointPath = null;
	for (int i = 0; i < 20; i++) {
		LOG.info("Triggering savepoint (" + (i + 1) + "/20).");
		try {
			savepointPath = clusterClient.triggerSavepoint(jobId, null)
				.get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS);
		} catch (Exception cause) {
			LOG.info("Failed to trigger savepoint. Retrying...", cause);
			// This can fail if the operators are not opened yet
			Thread.sleep(500);
		}
	}

	assertNotNull("Failed to trigger savepoint", savepointPath);

	clusterClient.disposeSavepoint(savepointPath).get();

	clusterClient.cancel(jobId).get();

	// make sure, the execution is finished to not influence other test methods
	invokeThread.join(deadline.timeLeft().toMillis());
	assertFalse("Program invoke thread still running", invokeThread.isAlive());
}
 

/**
 * Tests disposal of a savepoint, which contains custom user code KvState.
 */
@Test
public void testDisposeSavepointWithCustomKvState() throws Exception {
	ClusterClient<?> clusterClient = new MiniClusterClient(new Configuration(), miniClusterResource.getMiniCluster());

	Deadline deadline = new FiniteDuration(100, TimeUnit.SECONDS).fromNow();

	File checkpointDir = FOLDER.newFolder();
	File outputDir = FOLDER.newFolder();

	final PackagedProgram program = new PackagedProgram(
			new File(CUSTOM_KV_STATE_JAR_PATH),
			new String[] {
					String.valueOf(parallelism),
					checkpointDir.toURI().toString(),
					"5000",
					outputDir.toURI().toString()
			});

	TestStreamEnvironment.setAsContext(
		miniClusterResource.getMiniCluster(),
		parallelism,
		Collections.singleton(new Path(CUSTOM_KV_STATE_JAR_PATH)),
		Collections.<URL>emptyList()
	);

	// Execute detached
	Thread invokeThread = new Thread(new Runnable() {
		@Override
		public void run() {
			try {
				program.invokeInteractiveModeForExecution();
			} catch (ProgramInvocationException ignored) {
				if (ignored.getCause() == null ||
					!(ignored.getCause() instanceof JobCancellationException)) {
					ignored.printStackTrace();
				}
			}
		}
	});

	LOG.info("Starting program invoke thread");
	invokeThread.start();

	// The job ID
	JobID jobId = null;

	LOG.info("Waiting for job status running.");

	// Wait for running job
	while (jobId == null && deadline.hasTimeLeft()) {

		Collection<JobStatusMessage> jobs = clusterClient.listJobs().get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS);
		for (JobStatusMessage job : jobs) {
			if (job.getJobState() == JobStatus.RUNNING) {
				jobId = job.getJobId();
				LOG.info("Job running. ID: " + jobId);
				break;
			}
		}

		// Retry if job is not available yet
		if (jobId == null) {
			Thread.sleep(100L);
		}
	}

	// Trigger savepoint
	String savepointPath = null;
	for (int i = 0; i < 20; i++) {
		LOG.info("Triggering savepoint (" + (i + 1) + "/20).");
		try {
			savepointPath = clusterClient.triggerSavepoint(jobId, null)
				.get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS);
		} catch (Exception cause) {
			LOG.info("Failed to trigger savepoint. Retrying...", cause);
			// This can fail if the operators are not opened yet
			Thread.sleep(500);
		}
	}

	assertNotNull("Failed to trigger savepoint", savepointPath);

	clusterClient.disposeSavepoint(savepointPath).get();

	clusterClient.cancel(jobId);

	// make sure, the execution is finished to not influence other test methods
	invokeThread.join(deadline.timeLeft().toMillis());
	assertFalse("Program invoke thread still running", invokeThread.isAlive());
}