Java Code Examples for org.apache.flink.yarn.cli.FlinkYarnSessionCli#createClusterDescriptor()

The following examples show how to use org.apache.flink.yarn.cli.FlinkYarnSessionCli#createClusterDescriptor() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: FlinkYarnSessionCliTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testCorrectSettingOfMaxSlots() throws Exception {
	String[] params =
		new String[] {"-yn", "2", "-ys", "3"};

	FlinkYarnSessionCli yarnCLI = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final CommandLine commandLine = yarnCLI.parseCommandLineOptions(params, true);

	AbstractYarnClusterDescriptor descriptor = yarnCLI.createClusterDescriptor(commandLine);

	final ClusterSpecification clusterSpecification = yarnCLI.getClusterSpecification(commandLine);

	// each task manager has 3 slots but the parallelism is 7. Thus the slots should be increased.
	assertEquals(3, clusterSpecification.getSlotsPerTaskManager());
	assertEquals(2, clusterSpecification.getNumberTaskManagers());
}
Example 2
Source File: FlinkYarnSessionCliTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testCorrectSettingOfDetachedMode() throws Exception {
	String[] params =
		new String[] {"-yd"};

	FlinkYarnSessionCli yarnCLI = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final CommandLine commandLine = yarnCLI.parseCommandLineOptions(params, true);

	AbstractYarnClusterDescriptor descriptor = yarnCLI.createClusterDescriptor(commandLine);

	// each task manager has 3 slots but the parallelism is 7. Thus the slots should be increased.
	assertTrue(descriptor.isDetachedMode());
}
Example 3
Source File: FlinkYarnSessionCliTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testZookeeperNamespaceProperty() throws Exception {
	String zkNamespaceCliInput = "flink_test_namespace";

	String[] params = new String[] {"-yn", "2", "-yz", zkNamespaceCliInput};

	FlinkYarnSessionCli yarnCLI = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	CommandLine commandLine = yarnCLI.parseCommandLineOptions(params, true);

	AbstractYarnClusterDescriptor descriptor = yarnCLI.createClusterDescriptor(commandLine);

	assertEquals(zkNamespaceCliInput, descriptor.getZookeeperNamespace());
}
Example 4
Source File: FlinkYarnSessionCliTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testNodeLabelProperty() throws Exception {
	String nodeLabelCliInput = "flink_test_nodelabel";

	String[] params = new String[] { "-yn", "2", "-ynl", nodeLabelCliInput };

	FlinkYarnSessionCli yarnCLI = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	CommandLine commandLine = yarnCLI.parseCommandLineOptions(params, true);

	AbstractYarnClusterDescriptor descriptor = yarnCLI.createClusterDescriptor(commandLine);

	assertEquals(nodeLabelCliInput, descriptor.getNodeLabel());
}
Example 5
Source File: FlinkYarnSessionCliTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testResumeFromYarnIDZookeeperNamespace() throws Exception {
	final Configuration configuration = new Configuration();
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		configuration,
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[] {"-yid", TEST_YARN_APPLICATION_ID.toString()}, true);

	final AbstractYarnClusterDescriptor clusterDescriptor = flinkYarnSessionCli.createClusterDescriptor(commandLine);

	final Configuration clusterDescriptorConfiguration = clusterDescriptor.getFlinkConfiguration();

	String zkNs = clusterDescriptorConfiguration.getValue(HighAvailabilityOptions.HA_CLUSTER_ID);
	assertTrue(zkNs.matches("application_\\d+_0042"));
}
Example 6
Source File: FlinkYarnSessionCliTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testResumeFromYarnIDZookeeperNamespaceOverride() throws Exception {
	final Configuration configuration = new Configuration();
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		configuration,
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final String overrideZkNamespace = "my_cluster";

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[] {"-yid", TEST_YARN_APPLICATION_ID.toString(), "-yz", overrideZkNamespace}, true);

	final AbstractYarnClusterDescriptor clusterDescriptor = flinkYarnSessionCli.createClusterDescriptor(commandLine);

	final Configuration clusterDescriptorConfiguration = clusterDescriptor.getFlinkConfiguration();

	final String clusterId = clusterDescriptorConfiguration.getValue(HighAvailabilityOptions.HA_CLUSTER_ID);
	assertEquals(overrideZkNamespace, clusterId);
}
Example 7
Source File: FlinkYarnSessionCliTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testMultipleYarnShipOptions() throws Exception {
	final String[] args = new String[]{"run", "--yarnship", tmp.newFolder().getAbsolutePath(), "--yarnship", tmp.newFolder().getAbsolutePath()};
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);

	AbstractYarnClusterDescriptor flinkYarnDescriptor = flinkYarnSessionCli.createClusterDescriptor(commandLine);

	assertEquals(2, flinkYarnDescriptor.shipFiles.size());

}
Example 8
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testCorrectSettingOfMaxSlots() throws Exception {
	String[] params =
		new String[] {"-yn", "2", "-ys", "3"};

	FlinkYarnSessionCli yarnCLI = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final CommandLine commandLine = yarnCLI.parseCommandLineOptions(params, true);

	AbstractYarnClusterDescriptor descriptor = yarnCLI.createClusterDescriptor(commandLine);

	final ClusterSpecification clusterSpecification = yarnCLI.getClusterSpecification(commandLine);

	// each task manager has 3 slots but the parallelism is 7. Thus the slots should be increased.
	assertEquals(3, clusterSpecification.getSlotsPerTaskManager());
	assertEquals(2, clusterSpecification.getNumberTaskManagers());
}
Example 9
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testCorrectSettingOfDetachedMode() throws Exception {
	String[] params =
		new String[] {"-yd"};

	FlinkYarnSessionCli yarnCLI = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final CommandLine commandLine = yarnCLI.parseCommandLineOptions(params, true);

	AbstractYarnClusterDescriptor descriptor = yarnCLI.createClusterDescriptor(commandLine);

	// each task manager has 3 slots but the parallelism is 7. Thus the slots should be increased.
	assertTrue(descriptor.isDetachedMode());
}
Example 10
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testZookeeperNamespaceProperty() throws Exception {
	String zkNamespaceCliInput = "flink_test_namespace";

	String[] params = new String[] {"-yn", "2", "-yz", zkNamespaceCliInput};

	FlinkYarnSessionCli yarnCLI = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	CommandLine commandLine = yarnCLI.parseCommandLineOptions(params, true);

	AbstractYarnClusterDescriptor descriptor = yarnCLI.createClusterDescriptor(commandLine);

	assertEquals(zkNamespaceCliInput, descriptor.getZookeeperNamespace());
}
Example 11
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testNodeLabelProperty() throws Exception {
	String nodeLabelCliInput = "flink_test_nodelabel";

	String[] params = new String[] { "-yn", "2", "-ynl", nodeLabelCliInput };

	FlinkYarnSessionCli yarnCLI = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	CommandLine commandLine = yarnCLI.parseCommandLineOptions(params, true);

	AbstractYarnClusterDescriptor descriptor = yarnCLI.createClusterDescriptor(commandLine);

	assertEquals(nodeLabelCliInput, descriptor.getNodeLabel());
}
Example 12
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testResumeFromYarnIDZookeeperNamespace() throws Exception {
	final Configuration configuration = new Configuration();
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		configuration,
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[] {"-yid", TEST_YARN_APPLICATION_ID.toString()}, true);

	final AbstractYarnClusterDescriptor clusterDescriptor = flinkYarnSessionCli.createClusterDescriptor(commandLine);

	final Configuration clusterDescriptorConfiguration = clusterDescriptor.getFlinkConfiguration();

	String zkNs = clusterDescriptorConfiguration.getValue(HighAvailabilityOptions.HA_CLUSTER_ID);
	assertTrue(zkNs.matches("application_\\d+_0042"));
}
Example 13
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testResumeFromYarnIDZookeeperNamespaceOverride() throws Exception {
	final Configuration configuration = new Configuration();
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		configuration,
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final String overrideZkNamespace = "my_cluster";

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[] {"-yid", TEST_YARN_APPLICATION_ID.toString(), "-yz", overrideZkNamespace}, true);

	final AbstractYarnClusterDescriptor clusterDescriptor = flinkYarnSessionCli.createClusterDescriptor(commandLine);

	final Configuration clusterDescriptorConfiguration = clusterDescriptor.getFlinkConfiguration();

	final String clusterId = clusterDescriptorConfiguration.getValue(HighAvailabilityOptions.HA_CLUSTER_ID);
	assertEquals(overrideZkNamespace, clusterId);
}
Example 14
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testMultipleYarnShipOptions() throws Exception {
	final String[] args = new String[]{"run", "--yarnship", tmp.newFolder().getAbsolutePath(), "--yarnship", tmp.newFolder().getAbsolutePath()};
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);

	AbstractYarnClusterDescriptor flinkYarnDescriptor = flinkYarnSessionCli.createClusterDescriptor(commandLine);

	assertEquals(2, flinkYarnDescriptor.shipFiles.size());

}
Example 15
Source File: FlinkYarnSessionCliTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Test
public void testDynamicProperties() throws Exception {

	FlinkYarnSessionCli cli = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"",
		"",
		false);
	Options options = new Options();
	cli.addGeneralOptions(options);
	cli.addRunOptions(options);

	CommandLineParser parser = new DefaultParser();
	CommandLine cmd = parser.parse(options, new String[]{"run", "-j", "fake.jar", "-n", "15",
			"-D", "akka.ask.timeout=5 min", "-D", "env.java.opts=-DappName=foobar"});

	AbstractYarnClusterDescriptor flinkYarnDescriptor = cli.createClusterDescriptor(cmd);

	Assert.assertNotNull(flinkYarnDescriptor);

	Map<String, String> dynProperties =
		FlinkYarnSessionCli.getDynamicProperties(flinkYarnDescriptor.getDynamicPropertiesEncoded());
	assertEquals(2, dynProperties.size());
	assertEquals("5 min", dynProperties.get("akka.ask.timeout"));
	assertEquals("-DappName=foobar", dynProperties.get("env.java.opts"));
}
Example 16
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testDynamicProperties() throws Exception {

	FlinkYarnSessionCli cli = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"",
		"",
		false);
	Options options = new Options();
	cli.addGeneralOptions(options);
	cli.addRunOptions(options);

	CommandLineParser parser = new DefaultParser();
	CommandLine cmd = parser.parse(options, new String[]{"run", "-j", "fake.jar", "-n", "15",
			"-D", "akka.ask.timeout=5 min", "-D", "env.java.opts=-DappName=foobar"});

	AbstractYarnClusterDescriptor flinkYarnDescriptor = cli.createClusterDescriptor(cmd);

	Assert.assertNotNull(flinkYarnDescriptor);

	Map<String, String> dynProperties =
		FlinkYarnSessionCli.getDynamicProperties(flinkYarnDescriptor.getDynamicPropertiesEncoded());
	assertEquals(2, dynProperties.size());
	assertEquals("5 min", dynProperties.get("akka.ask.timeout"));
	assertEquals("-DappName=foobar", dynProperties.get("env.java.opts"));
}