Java Code Examples for org.apache.flink.yarn.cli.FlinkYarnSessionCli#parseCommandLineOptions()

/**
 * Tests that the configuration settings are used to create the
 * {@link ClusterSpecification}.
 */
@Test
public void testConfigurationClusterSpecification() throws Exception {
	final Configuration configuration = new Configuration();
	final int jobManagerMemory = 1337;
	configuration.setString(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY, jobManagerMemory + "m");
	final int taskManagerMemory = 7331;
	configuration.setString(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY, taskManagerMemory + "m");
	final int slotsPerTaskManager = 42;
	configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, slotsPerTaskManager);

	final String[] args = {};
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		configuration,
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);

	final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(jobManagerMemory));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(taskManagerMemory));
	assertThat(clusterSpecification.getSlotsPerTaskManager(), is(slotsPerTaskManager));
}
 

@Test
public void testZookeeperNamespaceProperty() throws Exception {
	String zkNamespaceCliInput = "flink_test_namespace";

	String[] params = new String[] {"-yn", "2", "-yz", zkNamespaceCliInput};

	FlinkYarnSessionCli yarnCLI = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	CommandLine commandLine = yarnCLI.parseCommandLineOptions(params, true);

	AbstractYarnClusterDescriptor descriptor = yarnCLI.createClusterDescriptor(commandLine);

	assertEquals(zkNamespaceCliInput, descriptor.getZookeeperNamespace());
}
 

@Test
public void testResumeFromYarnIDZookeeperNamespaceOverride() throws Exception {
	final FlinkYarnSessionCli flinkYarnSessionCli = createFlinkYarnSessionCli();

	final String overrideZkNamespace = "my_cluster";

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[] {"-yid", TEST_YARN_APPLICATION_ID.toString(), "-yz", overrideZkNamespace}, true);
	final Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
	final ClusterClientFactory<ApplicationId> clientFactory = getClusterClientFactory(executorConfig);

	final YarnClusterDescriptor clusterDescriptor = (YarnClusterDescriptor) clientFactory.createClusterDescriptor(executorConfig);

	final Configuration clusterDescriptorConfiguration = clusterDescriptor.getFlinkConfiguration();
	final String clusterId = clusterDescriptorConfiguration.getValue(HighAvailabilityOptions.HA_CLUSTER_ID);
	assertEquals(overrideZkNamespace, clusterId);
}
 

@Test
public void testCorrectSettingOfMaxSlots() throws Exception {
	String[] params =
		new String[] {"-yn", "2", "-ys", "3"};

	FlinkYarnSessionCli yarnCLI = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final CommandLine commandLine = yarnCLI.parseCommandLineOptions(params, true);

	AbstractYarnClusterDescriptor descriptor = yarnCLI.createClusterDescriptor(commandLine);

	final ClusterSpecification clusterSpecification = yarnCLI.getClusterSpecification(commandLine);

	// each task manager has 3 slots but the parallelism is 7. Thus the slots should be increased.
	assertEquals(3, clusterSpecification.getSlotsPerTaskManager());
	assertEquals(2, clusterSpecification.getNumberTaskManagers());
}
 

/**
 * Tests the specifying heap memory with config default value for job manager and task manager.
 */
@Test
public void testHeapMemoryPropertyWithConfigDefaultValue() throws Exception {
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[0], false);

	final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(1024));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(1024));
}
 

/**
 * Tests the specifying heap memory with config default value for job manager and task manager.
 */
@Test
public void testHeapMemoryPropertyWithConfigDefaultValue() throws Exception {
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[0], false);

	final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(1024));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(1024));
}
 

@Test
public void testMultipleYarnShipOptions() throws Exception {
	final String[] args = new String[]{"run", "--yarnship", tmp.newFolder().getAbsolutePath(), "--yarnship", tmp.newFolder().getAbsolutePath()};
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);

	AbstractYarnClusterDescriptor flinkYarnDescriptor = flinkYarnSessionCli.createClusterDescriptor(commandLine);

	assertEquals(2, flinkYarnDescriptor.shipFiles.size());

}
 

@Test
public void testNodeLabelProperty() throws Exception {
	String nodeLabelCliInput = "flink_test_nodelabel";

	String[] params = new String[] { "-yn", "2", "-ynl", nodeLabelCliInput };

	FlinkYarnSessionCli yarnCLI = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	CommandLine commandLine = yarnCLI.parseCommandLineOptions(params, true);

	AbstractYarnClusterDescriptor descriptor = yarnCLI.createClusterDescriptor(commandLine);

	assertEquals(nodeLabelCliInput, descriptor.getNodeLabel());
}
 

/**
 * Tests the specifying heap memory with old config key for job manager and task manager.
 */
@Test
public void testHeapMemoryPropertyWithOldConfigKey() throws Exception {
	Configuration configuration = new Configuration();
	configuration.setInteger(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY_MB, 2048);
	configuration.setInteger(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY_MB, 4096);

	final FlinkYarnSessionCli flinkYarnSessionCli = createFlinkYarnSessionCli(configuration);

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[0], false);

	final Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
	final ClusterClientFactory<ApplicationId> clientFactory = getClusterClientFactory(executorConfig);
	final ClusterSpecification clusterSpecification = clientFactory.getClusterSpecification(executorConfig);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(2048));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(4096));
}
 

/**
 * Tests that the command line arguments override the configuration settings
 * when the {@link ClusterSpecification} is created.
 */
@Test
public void testCommandLineClusterSpecification() throws Exception {
	final Configuration configuration = new Configuration();
	final int jobManagerMemory = 1337;
	final int taskManagerMemory = 7331;
	final int slotsPerTaskManager = 30;

	configuration.set(JobManagerOptions.TOTAL_PROCESS_MEMORY, MemorySize.ofMebiBytes(jobManagerMemory));
	configuration.set(TaskManagerOptions.TOTAL_PROCESS_MEMORY, MemorySize.ofMebiBytes(taskManagerMemory));
	configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, slotsPerTaskManager);

	final String[] args = {"-yjm", String.valueOf(jobManagerMemory) + "m", "-ytm", String.valueOf(taskManagerMemory) + "m", "-ys", String.valueOf(slotsPerTaskManager)};
	final FlinkYarnSessionCli flinkYarnSessionCli = createFlinkYarnSessionCli(configuration);

	CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);

	Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
	ClusterClientFactory<ApplicationId> clientFactory = getClusterClientFactory(executorConfig);
	ClusterSpecification clusterSpecification = clientFactory.getClusterSpecification(executorConfig);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(jobManagerMemory));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(taskManagerMemory));
	assertThat(clusterSpecification.getSlotsPerTaskManager(), is(slotsPerTaskManager));
}
 

/**
 * Test that the CliFrontend is able to pick up the .yarn-properties file from a specified location.
 */
@Test
public void testResumeFromYarnPropertiesFile() throws Exception {

	File directoryPath = writeYarnPropertiesFile(validPropertiesFile);

	final Configuration configuration = new Configuration();
	configuration.setString(YarnConfigOptions.PROPERTIES_FILE_LOCATION, directoryPath.getAbsolutePath());

	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		configuration,
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[] {}, true);

	final ApplicationId clusterId = flinkYarnSessionCli.getClusterId(commandLine);

	assertEquals(TEST_YARN_APPLICATION_ID, clusterId);
}
 

@Test
public void testZookeeperNamespaceProperty() throws Exception {
	String zkNamespaceCliInput = "flink_test_namespace";

	String[] params = new String[] {"-yn", "2", "-yz", zkNamespaceCliInput};

	FlinkYarnSessionCli yarnCLI = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	CommandLine commandLine = yarnCLI.parseCommandLineOptions(params, true);

	AbstractYarnClusterDescriptor descriptor = yarnCLI.createClusterDescriptor(commandLine);

	assertEquals(zkNamespaceCliInput, descriptor.getZookeeperNamespace());
}
 

@Test
public void testCorrectSettingOfDetachedMode() throws Exception {
	String[] params =
		new String[] {"-yd"};

	FlinkYarnSessionCli yarnCLI = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final CommandLine commandLine = yarnCLI.parseCommandLineOptions(params, true);

	AbstractYarnClusterDescriptor descriptor = yarnCLI.createClusterDescriptor(commandLine);

	// each task manager has 3 slots but the parallelism is 7. Thus the slots should be increased.
	assertTrue(descriptor.isDetachedMode());
}
 

/**
 * Tests the specifying total process memory without unit for job manager and task manager.
 */
@Test
public void testMemoryPropertyWithoutUnit() throws Exception {
	final String[] args = new String[] { "-yjm", "1024", "-ytm", "2048" };
	final FlinkYarnSessionCli flinkYarnSessionCli = createFlinkYarnSessionCli();

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);

	final Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
	final ClusterClientFactory<ApplicationId> clientFactory = getClusterClientFactory(executorConfig);
	final ClusterSpecification clusterSpecification = clientFactory.getClusterSpecification(executorConfig);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(1024));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(2048));
}
 

@Test
public void testResumeFromYarnIDZookeeperNamespace() throws Exception {
	final FlinkYarnSessionCli flinkYarnSessionCli = createFlinkYarnSessionCli();

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[] {"-yid", TEST_YARN_APPLICATION_ID.toString()}, true);

	final Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
	final ClusterClientFactory<ApplicationId> clientFactory = getClusterClientFactory(executorConfig);
	final YarnClusterDescriptor clusterDescriptor = (YarnClusterDescriptor) clientFactory.createClusterDescriptor(executorConfig);

	final Configuration clusterDescriptorConfiguration = clusterDescriptor.getFlinkConfiguration();

	String zkNs = clusterDescriptorConfiguration.getValue(HighAvailabilityOptions.HA_CLUSTER_ID);
	assertTrue(zkNs.matches("application_\\d+_0042"));
}
 

@Test
public void testMultipleYarnShipOptions() throws Exception {
	final String[] args = new String[]{"run", "--yarnship", tmp.newFolder().getAbsolutePath(), "--yarnship", tmp.newFolder().getAbsolutePath()};
	final FlinkYarnSessionCli flinkYarnSessionCli = createFlinkYarnSessionCli();

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);

	final Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
	final ClusterClientFactory<ApplicationId> clientFactory = getClusterClientFactory(executorConfig);
	YarnClusterDescriptor flinkYarnDescriptor = (YarnClusterDescriptor) clientFactory.createClusterDescriptor(executorConfig);

	assertEquals(2, flinkYarnDescriptor.getShipFiles().size());

}
 

/**
 * Tests the specifying heap memory with unit (MB) for job manager and task manager.
 */
@Test
public void testHeapMemoryPropertyWithUnitMB() throws Exception {
	final String[] args = new String[] { "-yn", "2", "-yjm", "1024m", "-ytm", "2048m" };
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");
	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);
	final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(1024));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(2048));
}
 

/**
 * Tests the specifying total process memory with unit (MB) for job manager and task manager.
 */
@Test
public void testMemoryPropertyWithUnitMB() throws Exception {
	final String[] args = new String[] { "-yjm", "1024m", "-ytm", "2048m" };
	final FlinkYarnSessionCli flinkYarnSessionCli = createFlinkYarnSessionCli();
	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);

	final Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
	final ClusterClientFactory<ApplicationId> clientFactory = getClusterClientFactory(executorConfig);
	final ClusterSpecification clusterSpecification = clientFactory.getClusterSpecification(executorConfig);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(1024));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(2048));
}
 

/**
 * Tests that the command line arguments override the configuration settings
 * when the {@link ClusterSpecification} is created.
 */
@Test
public void testCommandLineClusterSpecification() throws Exception {
	final Configuration configuration = new Configuration();
	final int jobManagerMemory = 1337;
	final int taskManagerMemory = 7331;
	final int slotsPerTaskManager = 30;

	configuration.setString(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY, jobManagerMemory + "m");
	configuration.setString(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY, taskManagerMemory + "m");
	configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, slotsPerTaskManager);

	final String[] args = {"-yjm", String.valueOf(jobManagerMemory) + "m", "-ytm", String.valueOf(taskManagerMemory) + "m", "-ys", String.valueOf(slotsPerTaskManager)};
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		configuration,
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);

	final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(jobManagerMemory));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(taskManagerMemory));
	assertThat(clusterSpecification.getSlotsPerTaskManager(), is(slotsPerTaskManager));
}
 

/**
 * Tests the specifying total process memory with arbitrary unit for job manager and task manager.
 */
@Test
public void testMemoryPropertyWithArbitraryUnit() throws Exception {
	final String[] args = new String[] { "-yjm", "1g", "-ytm", "2g" };
	final FlinkYarnSessionCli flinkYarnSessionCli = createFlinkYarnSessionCli();
	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);

	final Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
	final ClusterClientFactory<ApplicationId> clientFactory = getClusterClientFactory(executorConfig);
	final ClusterSpecification clusterSpecification = clientFactory.getClusterSpecification(executorConfig);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(1024));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(2048));
}