Java Code Examples for org.apache.flink.yarn.cli.FlinkYarnSessionCli#getClusterSpecification()

The following examples show how to use org.apache.flink.yarn.cli.FlinkYarnSessionCli#getClusterSpecification() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: FlinkYarnSessionCliTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testCorrectSettingOfMaxSlots() throws Exception {
	String[] params =
		new String[] {"-yn", "2", "-ys", "3"};

	FlinkYarnSessionCli yarnCLI = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final CommandLine commandLine = yarnCLI.parseCommandLineOptions(params, true);

	AbstractYarnClusterDescriptor descriptor = yarnCLI.createClusterDescriptor(commandLine);

	final ClusterSpecification clusterSpecification = yarnCLI.getClusterSpecification(commandLine);

	// each task manager has 3 slots but the parallelism is 7. Thus the slots should be increased.
	assertEquals(3, clusterSpecification.getSlotsPerTaskManager());
	assertEquals(2, clusterSpecification.getNumberTaskManagers());
}
Example 2
Source File: FlinkYarnSessionCliTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
/**
 * Tests that the configuration settings are used to create the
 * {@link ClusterSpecification}.
 */
@Test
public void testConfigurationClusterSpecification() throws Exception {
	final Configuration configuration = new Configuration();
	final int jobManagerMemory = 1337;
	configuration.setString(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY, jobManagerMemory + "m");
	final int taskManagerMemory = 7331;
	configuration.setString(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY, taskManagerMemory + "m");
	final int slotsPerTaskManager = 42;
	configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, slotsPerTaskManager);

	final String[] args = {};
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		configuration,
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);

	final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(jobManagerMemory));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(taskManagerMemory));
	assertThat(clusterSpecification.getSlotsPerTaskManager(), is(slotsPerTaskManager));
}
Example 3
Source File: FlinkYarnSessionCliTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
/**
 * Tests the specifying heap memory without unit for job manager and task manager.
 */
@Test
public void testHeapMemoryPropertyWithoutUnit() throws Exception {
	final String[] args = new String[] { "-yn", "2", "-yjm", "1024", "-ytm", "2048" };
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);

	final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(1024));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(2048));
}
Example 4
Source File: FlinkYarnSessionCliTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
/**
 * Tests the specifying heap memory with old config key for job manager and task manager.
 */
@Test
public void testHeapMemoryPropertyWithOldConfigKey() throws Exception {
	Configuration configuration = new Configuration();
	configuration.setInteger(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY_MB, 2048);
	configuration.setInteger(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY_MB, 4096);

	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		configuration,
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[0], false);

	final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(2048));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(4096));
}
Example 5
Source File: FlinkYarnSessionCliTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
/**
 * Tests the specifying heap memory with config default value for job manager and task manager.
 */
@Test
public void testHeapMemoryPropertyWithConfigDefaultValue() throws Exception {
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[0], false);

	final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(1024));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(1024));
}
Example 6
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testCorrectSettingOfMaxSlots() throws Exception {
	String[] params =
		new String[] {"-yn", "2", "-ys", "3"};

	FlinkYarnSessionCli yarnCLI = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final CommandLine commandLine = yarnCLI.parseCommandLineOptions(params, true);

	AbstractYarnClusterDescriptor descriptor = yarnCLI.createClusterDescriptor(commandLine);

	final ClusterSpecification clusterSpecification = yarnCLI.getClusterSpecification(commandLine);

	// each task manager has 3 slots but the parallelism is 7. Thus the slots should be increased.
	assertEquals(3, clusterSpecification.getSlotsPerTaskManager());
	assertEquals(2, clusterSpecification.getNumberTaskManagers());
}
Example 7
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Tests that the configuration settings are used to create the
 * {@link ClusterSpecification}.
 */
@Test
public void testConfigurationClusterSpecification() throws Exception {
	final Configuration configuration = new Configuration();
	final int jobManagerMemory = 1337;
	configuration.setString(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY, jobManagerMemory + "m");
	final int taskManagerMemory = 7331;
	configuration.setString(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY, taskManagerMemory + "m");
	final int slotsPerTaskManager = 42;
	configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, slotsPerTaskManager);

	final String[] args = {};
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		configuration,
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);

	final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(jobManagerMemory));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(taskManagerMemory));
	assertThat(clusterSpecification.getSlotsPerTaskManager(), is(slotsPerTaskManager));
}
Example 8
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Tests the specifying heap memory without unit for job manager and task manager.
 */
@Test
public void testHeapMemoryPropertyWithoutUnit() throws Exception {
	final String[] args = new String[] { "-yn", "2", "-yjm", "1024", "-ytm", "2048" };
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);

	final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(1024));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(2048));
}
Example 9
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Tests the specifying heap memory with old config key for job manager and task manager.
 */
@Test
public void testHeapMemoryPropertyWithOldConfigKey() throws Exception {
	Configuration configuration = new Configuration();
	configuration.setInteger(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY_MB, 2048);
	configuration.setInteger(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY_MB, 4096);

	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		configuration,
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[0], false);

	final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(2048));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(4096));
}
Example 10
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Tests the specifying heap memory with config default value for job manager and task manager.
 */
@Test
public void testHeapMemoryPropertyWithConfigDefaultValue() throws Exception {
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[0], false);

	final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(1024));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(1024));
}
Example 11
Source File: FlinkYarnSessionCliTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
/**
 * Tests that the command line arguments override the configuration settings
 * when the {@link ClusterSpecification} is created.
 */
@Test
public void testCommandLineClusterSpecification() throws Exception {
	final Configuration configuration = new Configuration();
	final int jobManagerMemory = 1337;
	final int taskManagerMemory = 7331;
	final int slotsPerTaskManager = 30;

	configuration.setString(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY, jobManagerMemory + "m");
	configuration.setString(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY, taskManagerMemory + "m");
	configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, slotsPerTaskManager);

	final String[] args = {"-yjm", String.valueOf(jobManagerMemory) + "m", "-ytm", String.valueOf(taskManagerMemory) + "m", "-ys", String.valueOf(slotsPerTaskManager)};
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		configuration,
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);

	final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(jobManagerMemory));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(taskManagerMemory));
	assertThat(clusterSpecification.getSlotsPerTaskManager(), is(slotsPerTaskManager));
}
Example 12
Source File: FlinkYarnSessionCliTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
/**
 * Tests the specifying heap memory with unit (MB) for job manager and task manager.
 */
@Test
public void testHeapMemoryPropertyWithUnitMB() throws Exception {
	final String[] args = new String[] { "-yn", "2", "-yjm", "1024m", "-ytm", "2048m" };
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");
	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);
	final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(1024));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(2048));
}
Example 13
Source File: FlinkYarnSessionCliTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
/**
 * Tests the specifying heap memory with arbitrary unit for job manager and task manager.
 */
@Test
public void testHeapMemoryPropertyWithArbitraryUnit() throws Exception {
	final String[] args = new String[] { "-yn", "2", "-yjm", "1g", "-ytm", "2g" };
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");
	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);
	final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(1024));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(2048));
}
Example 14
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Tests that the command line arguments override the configuration settings
 * when the {@link ClusterSpecification} is created.
 */
@Test
public void testCommandLineClusterSpecification() throws Exception {
	final Configuration configuration = new Configuration();
	final int jobManagerMemory = 1337;
	final int taskManagerMemory = 7331;
	final int slotsPerTaskManager = 30;

	configuration.setString(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY, jobManagerMemory + "m");
	configuration.setString(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY, taskManagerMemory + "m");
	configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, slotsPerTaskManager);

	final String[] args = {"-yjm", String.valueOf(jobManagerMemory) + "m", "-ytm", String.valueOf(taskManagerMemory) + "m", "-ys", String.valueOf(slotsPerTaskManager)};
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		configuration,
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);

	final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(jobManagerMemory));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(taskManagerMemory));
	assertThat(clusterSpecification.getSlotsPerTaskManager(), is(slotsPerTaskManager));
}
Example 15
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Tests the specifying heap memory with unit (MB) for job manager and task manager.
 */
@Test
public void testHeapMemoryPropertyWithUnitMB() throws Exception {
	final String[] args = new String[] { "-yn", "2", "-yjm", "1024m", "-ytm", "2048m" };
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");
	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);
	final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(1024));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(2048));
}
Example 16
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Tests the specifying heap memory with arbitrary unit for job manager and task manager.
 */
@Test
public void testHeapMemoryPropertyWithArbitraryUnit() throws Exception {
	final String[] args = new String[] { "-yn", "2", "-yjm", "1g", "-ytm", "2g" };
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");
	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);
	final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(1024));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(2048));
}