Java Code Examples for org.apache.flink.yarn.cli.FlinkYarnSessionCli#applyCommandLineOptionsToConfiguration()

@Test
public void testCorrectSettingOfMaxSlots() throws Exception {
	String[] params =
		new String[] {"-ys", "3"};

	FlinkYarnSessionCli yarnCLI = createFlinkYarnSessionCliWithJmAndTmTotalMemory(2048);

	final CommandLine commandLine = yarnCLI.parseCommandLineOptions(params, true);

	final Configuration executorConfig = yarnCLI.applyCommandLineOptionsToConfiguration(commandLine);
	final ClusterClientFactory<ApplicationId> clientFactory = getClusterClientFactory(executorConfig);
	final ClusterSpecification clusterSpecification = clientFactory.getClusterSpecification(executorConfig);

	// each task manager has 3 slots but the parallelism is 7. Thus the slots should be increased.
	assertEquals(3, clusterSpecification.getSlotsPerTaskManager());
}
 

/**
 * Tests the specifying heap memory with old config key for job manager and task manager.
 */
@Test
public void testHeapMemoryPropertyWithOldConfigKey() throws Exception {
	Configuration configuration = new Configuration();
	configuration.setInteger(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY_MB, 2048);
	configuration.setInteger(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY_MB, 4096);

	final FlinkYarnSessionCli flinkYarnSessionCli = createFlinkYarnSessionCli(configuration);

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[0], false);

	final Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
	final ClusterClientFactory<ApplicationId> clientFactory = getClusterClientFactory(executorConfig);
	final ClusterSpecification clusterSpecification = clientFactory.getClusterSpecification(executorConfig);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(2048));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(4096));
}
 

/**
 * Tests that the configuration settings are used to create the
 * {@link ClusterSpecification}.
 */
@Test
public void testConfigurationClusterSpecification() throws Exception {
	final Configuration configuration = new Configuration();
	final int jobManagerMemory = 1337;
	configuration.set(JobManagerOptions.TOTAL_PROCESS_MEMORY, MemorySize.ofMebiBytes(jobManagerMemory));
	final int taskManagerMemory = 7331;
	configuration.set(TaskManagerOptions.TOTAL_PROCESS_MEMORY, MemorySize.ofMebiBytes(taskManagerMemory));
	final int slotsPerTaskManager = 42;
	configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, slotsPerTaskManager);

	final String[] args = {};
	final FlinkYarnSessionCli flinkYarnSessionCli = createFlinkYarnSessionCli(configuration);

	CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);

	Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
	ClusterClientFactory<ApplicationId> clientFactory = getClusterClientFactory(executorConfig);
	ClusterSpecification clusterSpecification = clientFactory.getClusterSpecification(executorConfig);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(jobManagerMemory));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(taskManagerMemory));
	assertThat(clusterSpecification.getSlotsPerTaskManager(), is(slotsPerTaskManager));
}
 

/**
 * Tests that the command line arguments override the configuration settings
 * when the {@link ClusterSpecification} is created.
 */
@Test
public void testCommandLineClusterSpecification() throws Exception {
	final Configuration configuration = new Configuration();
	final int jobManagerMemory = 1337;
	final int taskManagerMemory = 7331;
	final int slotsPerTaskManager = 30;

	configuration.set(JobManagerOptions.TOTAL_PROCESS_MEMORY, MemorySize.ofMebiBytes(jobManagerMemory));
	configuration.set(TaskManagerOptions.TOTAL_PROCESS_MEMORY, MemorySize.ofMebiBytes(taskManagerMemory));
	configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, slotsPerTaskManager);

	final String[] args = {"-yjm", String.valueOf(jobManagerMemory) + "m", "-ytm", String.valueOf(taskManagerMemory) + "m", "-ys", String.valueOf(slotsPerTaskManager)};
	final FlinkYarnSessionCli flinkYarnSessionCli = createFlinkYarnSessionCli(configuration);

	CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);

	Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
	ClusterClientFactory<ApplicationId> clientFactory = getClusterClientFactory(executorConfig);
	ClusterSpecification clusterSpecification = clientFactory.getClusterSpecification(executorConfig);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(jobManagerMemory));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(taskManagerMemory));
	assertThat(clusterSpecification.getSlotsPerTaskManager(), is(slotsPerTaskManager));
}
 

@Test
public void testYarnIDOverridesPropertiesFile() throws Exception {
	File directoryPath = writeYarnPropertiesFile(validPropertiesFile);

	final Configuration configuration = new Configuration();
	configuration.setString(YarnConfigOptions.PROPERTIES_FILE_LOCATION, directoryPath.getAbsolutePath());

	final FlinkYarnSessionCli flinkYarnSessionCli = createFlinkYarnSessionCli(configuration);
	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[] {"-yid", TEST_YARN_APPLICATION_ID_2.toString() }, true);

	final Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
	final ClusterClientFactory<ApplicationId> clientFactory = getClusterClientFactory(executorConfig);
	final ApplicationId clusterId = clientFactory.getClusterId(executorConfig);

	assertEquals(TEST_YARN_APPLICATION_ID_2, clusterId);
}
 

@Test
public void testResumeFromYarnIDZookeeperNamespaceOverride() throws Exception {
	final FlinkYarnSessionCli flinkYarnSessionCli = createFlinkYarnSessionCli();

	final String overrideZkNamespace = "my_cluster";

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[] {"-yid", TEST_YARN_APPLICATION_ID.toString(), "-yz", overrideZkNamespace}, true);
	final Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
	final ClusterClientFactory<ApplicationId> clientFactory = getClusterClientFactory(executorConfig);

	final YarnClusterDescriptor clusterDescriptor = (YarnClusterDescriptor) clientFactory.createClusterDescriptor(executorConfig);

	final Configuration clusterDescriptorConfiguration = clusterDescriptor.getFlinkConfiguration();
	final String clusterId = clusterDescriptorConfiguration.getValue(HighAvailabilityOptions.HA_CLUSTER_ID);
	assertEquals(overrideZkNamespace, clusterId);
}
 

@Test
public void testDynamicProperties() throws Exception {

	FlinkYarnSessionCli cli = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"",
		"",
		false);
	Options options = new Options();
	cli.addGeneralOptions(options);
	cli.addRunOptions(options);

	CommandLineParser parser = new DefaultParser();
	CommandLine cmd = parser.parse(options, new String[]{"run", "-j", "fake.jar",
			"-D", AkkaOptions.ASK_TIMEOUT.key() + "=5 min",
			"-D", CoreOptions.FLINK_JVM_OPTIONS.key() + "=-DappName=foobar",
			"-D", SecurityOptions.SSL_INTERNAL_KEY_PASSWORD.key() + "=changeit"});

	Configuration executorConfig = cli.applyCommandLineOptionsToConfiguration(cmd);
	assertEquals("5 min", executorConfig.get(AkkaOptions.ASK_TIMEOUT));
	assertEquals("-DappName=foobar", executorConfig.get(CoreOptions.FLINK_JVM_OPTIONS));
	assertEquals("changeit", executorConfig.get(SecurityOptions.SSL_INTERNAL_KEY_PASSWORD));
}
 

@Test
public void testResumeFromYarnID() throws Exception {
	final FlinkYarnSessionCli flinkYarnSessionCli = createFlinkYarnSessionCli();

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[] {"-yid", TEST_YARN_APPLICATION_ID.toString()}, true);

	final Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
	final ClusterClientFactory<ApplicationId> clientFactory = getClusterClientFactory(executorConfig);
	final ApplicationId clusterId = clientFactory.getClusterId(executorConfig);

	assertEquals(TEST_YARN_APPLICATION_ID, clusterId);
}
 

@Test
public void testResumeFromYarnIDZookeeperNamespace() throws Exception {
	final FlinkYarnSessionCli flinkYarnSessionCli = createFlinkYarnSessionCli();

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[] {"-yid", TEST_YARN_APPLICATION_ID.toString()}, true);

	final Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
	final ClusterClientFactory<ApplicationId> clientFactory = getClusterClientFactory(executorConfig);
	final YarnClusterDescriptor clusterDescriptor = (YarnClusterDescriptor) clientFactory.createClusterDescriptor(executorConfig);

	final Configuration clusterDescriptorConfiguration = clusterDescriptor.getFlinkConfiguration();

	String zkNs = clusterDescriptorConfiguration.getValue(HighAvailabilityOptions.HA_CLUSTER_ID);
	assertTrue(zkNs.matches("application_\\d+_0042"));
}
 

@Test
public void testCorrectSettingOfDetachedMode() throws Exception {
	final String[] params = new String[] {"-yd"};
	FlinkYarnSessionCli yarnCLI = createFlinkYarnSessionCli();

	final CommandLine commandLine = yarnCLI.parseCommandLineOptions(params, true);
	final Configuration executorConfig = yarnCLI.applyCommandLineOptionsToConfiguration(commandLine);

	assertThat(executorConfig.get(DeploymentOptions.ATTACHED), is(false));
}
 

/**
 * Tests the specifying total process memory without unit for job manager and task manager.
 */
@Test
public void testMemoryPropertyWithoutUnit() throws Exception {
	final String[] args = new String[] { "-yjm", "1024", "-ytm", "2048" };
	final FlinkYarnSessionCli flinkYarnSessionCli = createFlinkYarnSessionCli();

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);

	final Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
	final ClusterClientFactory<ApplicationId> clientFactory = getClusterClientFactory(executorConfig);
	final ClusterSpecification clusterSpecification = clientFactory.getClusterSpecification(executorConfig);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(1024));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(2048));
}
 

/**
 * Tests the specifying total process memory with unit (MB) for job manager and task manager.
 */
@Test
public void testMemoryPropertyWithUnitMB() throws Exception {
	final String[] args = new String[] { "-yjm", "1024m", "-ytm", "2048m" };
	final FlinkYarnSessionCli flinkYarnSessionCli = createFlinkYarnSessionCli();
	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);

	final Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
	final ClusterClientFactory<ApplicationId> clientFactory = getClusterClientFactory(executorConfig);
	final ClusterSpecification clusterSpecification = clientFactory.getClusterSpecification(executorConfig);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(1024));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(2048));
}
 

/**
 * Tests the specifying total process memory with arbitrary unit for job manager and task manager.
 */
@Test
public void testMemoryPropertyWithArbitraryUnit() throws Exception {
	final String[] args = new String[] { "-yjm", "1g", "-ytm", "2g" };
	final FlinkYarnSessionCli flinkYarnSessionCli = createFlinkYarnSessionCli();
	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);

	final Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
	final ClusterClientFactory<ApplicationId> clientFactory = getClusterClientFactory(executorConfig);
	final ClusterSpecification clusterSpecification = clientFactory.getClusterSpecification(executorConfig);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(1024));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(2048));
}
 

/**
 * Tests the specifying job manager total process memory with config default value for job manager and task manager.
 */
@Test
public void testJobManagerMemoryPropertyWithConfigDefaultValue() throws Exception {
	int procMemory = 2048;
	final FlinkYarnSessionCli flinkYarnSessionCli = createFlinkYarnSessionCliWithJmAndTmTotalMemory(procMemory);

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[0], false);

	final Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
	final ClusterClientFactory<ApplicationId> clientFactory = getClusterClientFactory(executorConfig);
	final ClusterSpecification clusterSpecification = clientFactory.getClusterSpecification(executorConfig);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(procMemory));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(procMemory));
}
 

@Test
public void testMultipleYarnShipOptions() throws Exception {
	final String[] args = new String[]{"run", "--yarnship", tmp.newFolder().getAbsolutePath(), "--yarnship", tmp.newFolder().getAbsolutePath()};
	final FlinkYarnSessionCli flinkYarnSessionCli = createFlinkYarnSessionCli();

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);

	final Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
	final ClusterClientFactory<ApplicationId> clientFactory = getClusterClientFactory(executorConfig);
	YarnClusterDescriptor flinkYarnDescriptor = (YarnClusterDescriptor) clientFactory.createClusterDescriptor(executorConfig);

	assertEquals(2, flinkYarnDescriptor.getShipFiles().size());

}
 

/**
 * Test that the CliFrontend is able to pick up the .yarn-properties file from a specified location.
 */
@Test
public void testResumeFromYarnPropertiesFile() throws Exception {

	File directoryPath = writeYarnPropertiesFile(validPropertiesFile);

	final Configuration configuration = new Configuration();
	configuration.setString(YarnConfigOptions.PROPERTIES_FILE_LOCATION, directoryPath.getAbsolutePath());

	final FlinkYarnSessionCli flinkYarnSessionCli = createFlinkYarnSessionCli(configuration);

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[] {}, true);

	final Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
	final ClusterClientFactory<ApplicationId> clientFactory = getClusterClientFactory(executorConfig);
	final ApplicationId clusterId = clientFactory.getClusterId(executorConfig);

	assertEquals(TEST_YARN_APPLICATION_ID, clusterId);
}
 

@Test
public void testNodeLabelProperty() throws Exception {
	String nodeLabelCliInput = "flink_test_nodelabel";

	String[] params = new String[] {"-ynl", nodeLabelCliInput };

	FlinkYarnSessionCli yarnCLI = createFlinkYarnSessionCli();

	CommandLine commandLine = yarnCLI.parseCommandLineOptions(params, true);

	Configuration executorConfig = yarnCLI.applyCommandLineOptionsToConfiguration(commandLine);
	ClusterClientFactory<ApplicationId> clientFactory = getClusterClientFactory(executorConfig);
	YarnClusterDescriptor descriptor = (YarnClusterDescriptor) clientFactory.createClusterDescriptor(executorConfig);

	assertEquals(nodeLabelCliInput, descriptor.getNodeLabel());
}
 

@Test
public void testZookeeperNamespaceProperty() throws Exception {
	String zkNamespaceCliInput = "flink_test_namespace";

	String[] params = new String[] {"-yz", zkNamespaceCliInput};

	FlinkYarnSessionCli yarnCLI = createFlinkYarnSessionCli();

	CommandLine commandLine = yarnCLI.parseCommandLineOptions(params, true);

	Configuration executorConfig = yarnCLI.applyCommandLineOptionsToConfiguration(commandLine);
	ClusterClientFactory<ApplicationId> clientFactory = getClusterClientFactory(executorConfig);
	YarnClusterDescriptor descriptor = (YarnClusterDescriptor) clientFactory.createClusterDescriptor(executorConfig);

	assertEquals(zkNamespaceCliInput, descriptor.getZookeeperNamespace());
}