org.apache.flink.configuration.ConfigurationUtils Java Examples
The following examples show how to use
org.apache.flink.configuration.ConfigurationUtils.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TaskManagerRunner.java From flink with Apache License 2.0 | 6 votes |
@VisibleForTesting static Configuration loadConfiguration(String[] args) throws FlinkParseException { final CommandLineParser<ClusterConfiguration> commandLineParser = new CommandLineParser<>(new ClusterConfigurationParserFactory()); final ClusterConfiguration clusterConfiguration; try { clusterConfiguration = commandLineParser.parse(args); } catch (FlinkParseException e) { LOG.error("Could not parse the command line options.", e); commandLineParser.printHelp(TaskManagerRunner.class.getSimpleName()); throw e; } final Configuration dynamicProperties = ConfigurationUtils.createConfiguration(clusterConfiguration.getDynamicProperties()); return GlobalConfiguration.loadConfiguration(clusterConfiguration.getConfigDir(), dynamicProperties); }
Example #2
Source File: SavepointEnvironment.java From flink with Apache License 2.0 | 6 votes |
private SavepointEnvironment(RuntimeContext ctx, Configuration configuration, int maxParallelism, int indexOfSubtask, PrioritizedOperatorSubtaskState prioritizedOperatorSubtaskState) { this.jobID = new JobID(); this.vertexID = new JobVertexID(); this.attemptID = new ExecutionAttemptID(); this.ctx = Preconditions.checkNotNull(ctx); this.configuration = Preconditions.checkNotNull(configuration); Preconditions.checkArgument(maxParallelism > 0 && indexOfSubtask < maxParallelism); this.maxParallelism = maxParallelism; this.indexOfSubtask = indexOfSubtask; this.registry = new KvStateRegistry().createTaskRegistry(jobID, vertexID); this.taskStateManager = new SavepointTaskStateManager(prioritizedOperatorSubtaskState); this.ioManager = new IOManagerAsync(ConfigurationUtils.parseTempDirectories(configuration)); this.memoryManager = MemoryManager.forDefaultPageSize(64 * 1024 * 1024); this.accumulatorRegistry = new AccumulatorRegistry(jobID, attemptID); }
Example #3
Source File: JobMasterConfiguration.java From flink with Apache License 2.0 | 6 votes |
public static JobMasterConfiguration fromConfiguration(Configuration configuration) { final Time rpcTimeout = AkkaUtils.getTimeoutAsTime(configuration); final Time slotRequestTimeout = Time.milliseconds(configuration.getLong(JobManagerOptions.SLOT_REQUEST_TIMEOUT)); final String tmpDirectory = ConfigurationUtils.parseTempDirectories(configuration)[0]; final RetryingRegistrationConfiguration retryingRegistrationConfiguration = RetryingRegistrationConfiguration.fromConfiguration(configuration); return new JobMasterConfiguration( rpcTimeout, slotRequestTimeout, tmpDirectory, retryingRegistrationConfiguration, configuration); }
Example #4
Source File: ClusterEntrypoint.java From flink with Apache License 2.0 | 6 votes |
protected static Configuration loadConfiguration(EntrypointClusterConfiguration entrypointClusterConfiguration) { final Configuration dynamicProperties = ConfigurationUtils.createConfiguration(entrypointClusterConfiguration.getDynamicProperties()); final Configuration configuration = GlobalConfiguration.loadConfiguration(entrypointClusterConfiguration.getConfigDir(), dynamicProperties); final int restPort = entrypointClusterConfiguration.getRestPort(); if (restPort >= 0) { configuration.setInteger(RestOptions.PORT, restPort); } final String hostname = entrypointClusterConfiguration.getHostname(); if (hostname != null) { configuration.setString(JobManagerOptions.ADDRESS, hostname); } return configuration; }
Example #5
Source File: SessionClusterEntrypoint.java From flink with Apache License 2.0 | 6 votes |
@Override protected ArchivedExecutionGraphStore createSerializableExecutionGraphStore( Configuration configuration, ScheduledExecutor scheduledExecutor) throws IOException { final File tmpDir = new File(ConfigurationUtils.parseTempDirectories(configuration)[0]); final Time expirationTime = Time.seconds(configuration.getLong(JobManagerOptions.JOB_STORE_EXPIRATION_TIME)); final long maximumCacheSizeBytes = configuration.getLong(JobManagerOptions.JOB_STORE_CACHE_SIZE); return new FileArchivedExecutionGraphStore( tmpDir, expirationTime, maximumCacheSizeBytes, scheduledExecutor, Ticker.systemTicker()); }
Example #6
Source File: ClusterEntrypoint.java From flink with Apache License 2.0 | 6 votes |
protected static Configuration loadConfiguration(EntrypointClusterConfiguration entrypointClusterConfiguration) { final Configuration dynamicProperties = ConfigurationUtils.createConfiguration(entrypointClusterConfiguration.getDynamicProperties()); final Configuration configuration = GlobalConfiguration.loadConfiguration(entrypointClusterConfiguration.getConfigDir(), dynamicProperties); final int restPort = entrypointClusterConfiguration.getRestPort(); if (restPort >= 0) { configuration.setInteger(RestOptions.PORT, restPort); } final String hostname = entrypointClusterConfiguration.getHostname(); if (hostname != null) { configuration.setString(JobManagerOptions.ADDRESS, hostname); } return configuration; }
Example #7
Source File: SessionClusterEntrypoint.java From flink with Apache License 2.0 | 6 votes |
@Override protected ArchivedExecutionGraphStore createSerializableExecutionGraphStore( Configuration configuration, ScheduledExecutor scheduledExecutor) throws IOException { final File tmpDir = new File(ConfigurationUtils.parseTempDirectories(configuration)[0]); final Time expirationTime = Time.seconds(configuration.getLong(JobManagerOptions.JOB_STORE_EXPIRATION_TIME)); final int maximumCapacity = configuration.getInteger(JobManagerOptions.JOB_STORE_MAX_CAPACITY); final long maximumCacheSizeBytes = configuration.getLong(JobManagerOptions.JOB_STORE_CACHE_SIZE); return new FileArchivedExecutionGraphStore( tmpDir, expirationTime, maximumCapacity, maximumCacheSizeBytes, scheduledExecutor, Ticker.systemTicker()); }
Example #8
Source File: ConfigurationParserUtils.java From flink with Apache License 2.0 | 6 votes |
/** * Generate configuration from only the config file and dynamic properties. * @param args the commandline arguments * @param cmdLineSyntax the syntax for this application * @return generated configuration * @throws FlinkParseException if the configuration cannot be generated */ public static Configuration loadCommonConfiguration(String[] args, String cmdLineSyntax) throws FlinkParseException { final CommandLineParser<ClusterConfiguration> commandLineParser = new CommandLineParser<>(new ClusterConfigurationParserFactory()); final ClusterConfiguration clusterConfiguration; try { clusterConfiguration = commandLineParser.parse(args); } catch (FlinkParseException e) { LOG.error("Could not parse the command line options.", e); commandLineParser.printHelp(cmdLineSyntax); throw e; } final Configuration dynamicProperties = ConfigurationUtils.createConfiguration(clusterConfiguration.getDynamicProperties()); return GlobalConfiguration.loadConfiguration(clusterConfiguration.getConfigDir(), dynamicProperties); }
Example #9
Source File: JobMasterConfiguration.java From flink with Apache License 2.0 | 6 votes |
public static JobMasterConfiguration fromConfiguration(Configuration configuration) { final Time rpcTimeout = AkkaUtils.getTimeoutAsTime(configuration); final Time slotRequestTimeout = Time.milliseconds(configuration.getLong(JobManagerOptions.SLOT_REQUEST_TIMEOUT)); final String tmpDirectory = ConfigurationUtils.parseTempDirectories(configuration)[0]; final RetryingRegistrationConfiguration retryingRegistrationConfiguration = RetryingRegistrationConfiguration.fromConfiguration(configuration); return new JobMasterConfiguration( rpcTimeout, slotRequestTimeout, tmpDirectory, retryingRegistrationConfiguration, configuration); }
Example #10
Source File: SessionClusterEntrypoint.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Override protected ArchivedExecutionGraphStore createSerializableExecutionGraphStore( Configuration configuration, ScheduledExecutor scheduledExecutor) throws IOException { final File tmpDir = new File(ConfigurationUtils.parseTempDirectories(configuration)[0]); final Time expirationTime = Time.seconds(configuration.getLong(JobManagerOptions.JOB_STORE_EXPIRATION_TIME)); final long maximumCacheSizeBytes = configuration.getLong(JobManagerOptions.JOB_STORE_CACHE_SIZE); return new FileArchivedExecutionGraphStore( tmpDir, expirationTime, maximumCacheSizeBytes, scheduledExecutor, Ticker.systemTicker()); }
Example #11
Source File: ClusterEntrypoint.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
protected static Configuration loadConfiguration(EntrypointClusterConfiguration entrypointClusterConfiguration) { final Configuration dynamicProperties = ConfigurationUtils.createConfiguration(entrypointClusterConfiguration.getDynamicProperties()); final Configuration configuration = GlobalConfiguration.loadConfiguration(entrypointClusterConfiguration.getConfigDir(), dynamicProperties); final int restPort = entrypointClusterConfiguration.getRestPort(); if (restPort >= 0) { configuration.setInteger(RestOptions.PORT, restPort); } final String hostname = entrypointClusterConfiguration.getHostname(); if (hostname != null) { configuration.setString(JobManagerOptions.ADDRESS, hostname); } return configuration; }
Example #12
Source File: TaskManagerRunner.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@VisibleForTesting static Configuration loadConfiguration(String[] args) throws FlinkParseException { final CommandLineParser<ClusterConfiguration> commandLineParser = new CommandLineParser<>(new ClusterConfigurationParserFactory()); final ClusterConfiguration clusterConfiguration; try { clusterConfiguration = commandLineParser.parse(args); } catch (FlinkParseException e) { LOG.error("Could not parse the command line options.", e); commandLineParser.printHelp(TaskManagerRunner.class.getSimpleName()); throw e; } final Configuration dynamicProperties = ConfigurationUtils.createConfiguration(clusterConfiguration.getDynamicProperties()); return GlobalConfiguration.loadConfiguration(clusterConfiguration.getConfigDir(), dynamicProperties); }
Example #13
Source File: JobMasterConfiguration.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
public static JobMasterConfiguration fromConfiguration(Configuration configuration) { final Time rpcTimeout = AkkaUtils.getTimeoutAsTime(configuration); final Time slotRequestTimeout = Time.milliseconds(configuration.getLong(JobManagerOptions.SLOT_REQUEST_TIMEOUT)); final String tmpDirectory = ConfigurationUtils.parseTempDirectories(configuration)[0]; final RetryingRegistrationConfiguration retryingRegistrationConfiguration = RetryingRegistrationConfiguration.fromConfiguration(configuration); return new JobMasterConfiguration( rpcTimeout, slotRequestTimeout, tmpDirectory, retryingRegistrationConfiguration, configuration); }
Example #14
Source File: BootstrapToolsTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testGetEnvironmentVariables() { Configuration testConf = new Configuration(); testConf.setString("containerized.master.env.LD_LIBRARY_PATH", "/usr/lib/native"); Map<String, String> res = ConfigurationUtils.getPrefixedKeyValuePairs("containerized.master.env.", testConf); Assert.assertEquals(1, res.size()); Map.Entry<String, String> entry = res.entrySet().iterator().next(); Assert.assertEquals("LD_LIBRARY_PATH", entry.getKey()); Assert.assertEquals("/usr/lib/native", entry.getValue()); }
Example #15
Source File: TaskExecutorProcessUtilsTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testGenerateDynamicConfigurations() { String dynamicConfigsStr = TaskExecutorProcessUtils.generateDynamicConfigsStr(TM_RESOURCE_SPEC); Map<String, String> configs = ConfigurationUtils.parseTmResourceDynamicConfigs(dynamicConfigsStr); assertThat(new CPUResource(Double.valueOf(configs.get(TaskManagerOptions.CPU_CORES.key()))), is(TM_RESOURCE_SPEC.getCpuCores())); assertThat(MemorySize.parse(configs.get(TaskManagerOptions.FRAMEWORK_HEAP_MEMORY.key())), is(TM_RESOURCE_SPEC.getFrameworkHeapSize())); assertThat(MemorySize.parse(configs.get(TaskManagerOptions.FRAMEWORK_OFF_HEAP_MEMORY.key())), is(TM_RESOURCE_SPEC.getFrameworkOffHeapMemorySize())); assertThat(MemorySize.parse(configs.get(TaskManagerOptions.TASK_HEAP_MEMORY.key())), is(TM_RESOURCE_SPEC.getTaskHeapSize())); assertThat(MemorySize.parse(configs.get(TaskManagerOptions.TASK_OFF_HEAP_MEMORY.key())), is(TM_RESOURCE_SPEC.getTaskOffHeapSize())); assertThat(MemorySize.parse(configs.get(TaskManagerOptions.NETWORK_MEMORY_MAX.key())), is(TM_RESOURCE_SPEC.getNetworkMemSize())); assertThat(MemorySize.parse(configs.get(TaskManagerOptions.NETWORK_MEMORY_MIN.key())), is(TM_RESOURCE_SPEC.getNetworkMemSize())); assertThat(MemorySize.parse(configs.get(TaskManagerOptions.MANAGED_MEMORY_SIZE.key())), is(TM_RESOURCE_SPEC.getManagedMemorySize())); }
Example #16
Source File: BootstrapToolsTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testGetEnvironmentVariablesErroneous() { Configuration testConf = new Configuration(); testConf.setString("containerized.master.env.", "/usr/lib/native"); Map<String, String> res = ConfigurationUtils.getPrefixedKeyValuePairs("containerized.master.env.", testConf); Assert.assertEquals(0, res.size()); }
Example #17
Source File: BashJavaUtilsITCase.java From flink with Apache License 2.0 | 5 votes |
@Test public void testGetJmResourceParamsWithDynamicProperties() throws Exception { int expectedResultLines = 1; long metaspace = 123456789L; String[] commands = { RUN_BASH_JAVA_UTILS_CMD_SCRIPT, BashJavaUtils.Command.GET_JM_RESOURCE_PARAMS.toString(), String.valueOf(expectedResultLines), "-D" + JobManagerOptions.JVM_METASPACE.key() + "=" + metaspace + "b"}; List<String> lines = Arrays.asList(executeScript(commands).split(System.lineSeparator())); assertThat(lines.size(), is(expectedResultLines)); Map<String, String> params = ConfigurationUtils.parseJvmArgString(lines.get(0)); assertThat(Long.valueOf(params.get("-XX:MaxMetaspaceSize=")), is(metaspace)); }
Example #18
Source File: BashJavaUtilsTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testJmLegacyHeapOptionSetsNewJvmHeap() { Configuration configuration = new Configuration(); MemorySize heapSize = MemorySize.ofMebiBytes(10); configuration.set(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY, heapSize); String jvmArgsLine = BashJavaUtils.getJmResourceParams(configuration).get(0); Map<String, String> jvmArgs = ConfigurationUtils.parseJvmArgString(jvmArgsLine); String heapSizeStr = Long.toString(heapSize.getBytes()); assertThat(jvmArgs.get("-Xmx"), is(heapSizeStr)); assertThat(jvmArgs.get("-Xms"), is(heapSizeStr)); }
Example #19
Source File: ProcessMemoryUtilsTestBase.java From flink with Apache License 2.0 | 5 votes |
@Test public void testGenerateJvmParameters() { ProcessMemorySpec spec = JvmArgTestingProcessMemorySpec.generate(); String jvmParamsStr = ProcessMemoryUtils.generateJvmParametersStr(spec, true); Map<String, String> configs = ConfigurationUtils.parseJvmArgString(jvmParamsStr); assertThat(configs.size(), is(4)); assertThat(MemorySize.parse(configs.get("-Xmx")), is(spec.getJvmHeapMemorySize())); assertThat(MemorySize.parse(configs.get("-Xms")), is(spec.getJvmHeapMemorySize())); assertThat(MemorySize.parse(configs.get("-XX:MaxMetaspaceSize=")), is(spec.getJvmMetaspaceSize())); assertThat(MemorySize.parse(configs.get("-XX:MaxDirectMemorySize=")), is(spec.getJvmDirectMemorySize())); }
Example #20
Source File: ClusterEntrypoint.java From flink with Apache License 2.0 | 5 votes |
protected void initializeServices(Configuration configuration, PluginManager pluginManager) throws Exception { LOG.info("Initializing cluster services."); synchronized (lock) { commonRpcService = AkkaRpcServiceUtils.createRemoteRpcService( configuration, configuration.getString(JobManagerOptions.ADDRESS), getRPCPortRange(configuration), configuration.getString(JobManagerOptions.BIND_HOST), configuration.getOptional(JobManagerOptions.RPC_BIND_PORT)); // update the configuration used to create the high availability services configuration.setString(JobManagerOptions.ADDRESS, commonRpcService.getAddress()); configuration.setInteger(JobManagerOptions.PORT, commonRpcService.getPort()); ioExecutor = Executors.newFixedThreadPool( ClusterEntrypointUtils.getPoolSize(configuration), new ExecutorThreadFactory("cluster-io")); haServices = createHaServices(configuration, ioExecutor); blobServer = new BlobServer(configuration, haServices.createBlobStore()); blobServer.start(); heartbeatServices = createHeartbeatServices(configuration); metricRegistry = createMetricRegistry(configuration, pluginManager); final RpcService metricQueryServiceRpcService = MetricUtils.startRemoteMetricsRpcService(configuration, commonRpcService.getAddress()); metricRegistry.startQueryService(metricQueryServiceRpcService, null); final String hostname = RpcUtils.getHostname(commonRpcService); processMetricGroup = MetricUtils.instantiateProcessMetricGroup( metricRegistry, hostname, ConfigurationUtils.getSystemResourceMetricsProbingInterval(configuration)); archivedExecutionGraphStore = createSerializableExecutionGraphStore(configuration, commonRpcService.getScheduledExecutor()); } }
Example #21
Source File: ClusterConfigurationInfo.java From flink with Apache License 2.0 | 5 votes |
public static ClusterConfigurationInfo from(Configuration config) { final ClusterConfigurationInfo clusterConfig = new ClusterConfigurationInfo(config.keySet().size()); final Map<String, String> configurationWithHiddenSensitiveValues = ConfigurationUtils.hideSensitiveValues(config.toMap()); for (Map.Entry<String, String> keyValuePair : configurationWithHiddenSensitiveValues.entrySet()) { clusterConfig.add(new ClusterConfigurationInfoEntry(keyValuePair.getKey(), keyValuePair.getValue())); } return clusterConfig; }
Example #22
Source File: JobConfigInfo.java From flink with Apache License 2.0 | 5 votes |
public static ExecutionConfigInfo from(ArchivedExecutionConfig archivedExecutionConfig) { return new ExecutionConfigInfo( archivedExecutionConfig.getExecutionMode(), archivedExecutionConfig.getRestartStrategyDescription(), archivedExecutionConfig.getParallelism(), archivedExecutionConfig.getObjectReuseEnabled(), ConfigurationUtils.hideSensitiveValues(archivedExecutionConfig.getGlobalJobParameters())); }
Example #23
Source File: StandaloneResourceManagerFactory.java From flink with Apache License 2.0 | 5 votes |
@Override protected ResourceManager<ResourceID> createResourceManager( Configuration configuration, ResourceID resourceId, RpcService rpcService, HighAvailabilityServices highAvailabilityServices, HeartbeatServices heartbeatServices, FatalErrorHandler fatalErrorHandler, ClusterInformation clusterInformation, @Nullable String webInterfaceUrl, ResourceManagerMetricGroup resourceManagerMetricGroup, ResourceManagerRuntimeServices resourceManagerRuntimeServices) { final Time standaloneClusterStartupPeriodTime = ConfigurationUtils.getStandaloneClusterStartupPeriodTime(configuration); return new StandaloneResourceManager( rpcService, resourceId, highAvailabilityServices, heartbeatServices, resourceManagerRuntimeServices.getSlotManager(), ResourceManagerPartitionTrackerImpl::new, resourceManagerRuntimeServices.getJobLeaderIdService(), clusterInformation, fatalErrorHandler, resourceManagerMetricGroup, standaloneClusterStartupPeriodTime, AkkaUtils.getTimeoutAsTime(configuration)); }
Example #24
Source File: ProcessMemoryUtilsTestBase.java From flink with Apache License 2.0 | 5 votes |
@Test public void testGenerateJvmParametersWithoutDirectMemoryLimit() { ProcessMemorySpec spec = JvmArgTestingProcessMemorySpec.generate(); String jvmParamsStr = ProcessMemoryUtils.generateJvmParametersStr(spec, false); Map<String, String> configs = ConfigurationUtils.parseJvmArgString(jvmParamsStr); assertThat(configs.size(), is(3)); assertThat(MemorySize.parse(configs.get("-Xmx")), is(spec.getJvmHeapMemorySize())); assertThat(MemorySize.parse(configs.get("-Xms")), is(spec.getJvmHeapMemorySize())); assertThat(MemorySize.parse(configs.get("-XX:MaxMetaspaceSize=")), is(spec.getJvmMetaspaceSize())); assertThat(configs.containsKey("-XX:MaxDirectMemorySize="), is(false)); }
Example #25
Source File: ActiveResourceManagerFactory.java From flink with Apache License 2.0 | 5 votes |
public static Configuration createActiveResourceManagerConfiguration(Configuration originalConfiguration) { final int taskManagerMemoryMB = ConfigurationUtils.getTaskManagerHeapMemory(originalConfiguration).getMebiBytes(); final long cutoffMB = ContaineredTaskManagerParameters.calculateCutoffMB(originalConfiguration, taskManagerMemoryMB); final long processMemoryBytes = (taskManagerMemoryMB - cutoffMB) << 20; // megabytes to bytes final long managedMemoryBytes = TaskManagerServices.getManagedMemoryFromProcessMemory(originalConfiguration, processMemoryBytes); final Configuration resourceManagerConfig = new Configuration(originalConfiguration); resourceManagerConfig.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, managedMemoryBytes + "b"); return resourceManagerConfig; }
Example #26
Source File: FlinkYarnSessionCli.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
private ClusterSpecification createClusterSpecification(Configuration configuration, CommandLine cmd) { if (cmd.hasOption(container.getOpt())) { // number of containers is required option! LOG.info("The argument {} is deprecated in will be ignored.", container.getOpt()); } // TODO: The number of task manager should be deprecated soon final int numberTaskManagers; if (cmd.hasOption(container.getOpt())) { numberTaskManagers = Integer.valueOf(cmd.getOptionValue(container.getOpt())); } else { numberTaskManagers = 1; } // JobManager Memory final int jobManagerMemoryMB = ConfigurationUtils.getJobManagerHeapMemory(configuration).getMebiBytes(); // Task Managers memory final int taskManagerMemoryMB = ConfigurationUtils.getTaskManagerHeapMemory(configuration).getMebiBytes(); int slotsPerTaskManager = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS); return new ClusterSpecification.ClusterSpecificationBuilder() .setMasterMemoryMB(jobManagerMemoryMB) .setTaskManagerMemoryMB(taskManagerMemoryMB) .setNumberTaskManagers(numberTaskManagers) .setSlotsPerTaskManager(slotsPerTaskManager) .createClusterSpecification(); }
Example #27
Source File: ClusterSpecification.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
public static ClusterSpecification fromConfiguration(Configuration configuration) { int slots = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS, 1); int jobManagerMemoryMb = ConfigurationUtils.getJobManagerHeapMemory(configuration).getMebiBytes(); int taskManagerMemoryMb = ConfigurationUtils.getTaskManagerHeapMemory(configuration).getMebiBytes(); return new ClusterSpecificationBuilder() .setMasterMemoryMB(jobManagerMemoryMb) .setTaskManagerMemoryMB(taskManagerMemoryMb) .setNumberTaskManagers(1) .setSlotsPerTaskManager(slots) .createClusterSpecification(); }
Example #28
Source File: FlinkYarnSessionCli.java From flink with Apache License 2.0 | 5 votes |
private ClusterSpecification createClusterSpecification(Configuration configuration, CommandLine cmd) { if (cmd.hasOption(container.getOpt())) { // number of containers is required option! LOG.info("The argument {} is deprecated in will be ignored.", container.getOpt()); } // TODO: The number of task manager should be deprecated soon final int numberTaskManagers; if (cmd.hasOption(container.getOpt())) { numberTaskManagers = Integer.valueOf(cmd.getOptionValue(container.getOpt())); } else { numberTaskManagers = 1; } // JobManager Memory final int jobManagerMemoryMB = ConfigurationUtils.getJobManagerHeapMemory(configuration).getMebiBytes(); // Task Managers memory final int taskManagerMemoryMB = ConfigurationUtils.getTaskManagerHeapMemory(configuration).getMebiBytes(); int slotsPerTaskManager = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS); return new ClusterSpecification.ClusterSpecificationBuilder() .setMasterMemoryMB(jobManagerMemoryMB) .setTaskManagerMemoryMB(taskManagerMemoryMB) .setNumberTaskManagers(numberTaskManagers) .setSlotsPerTaskManager(slotsPerTaskManager) .createClusterSpecification(); }
Example #29
Source File: ClusterSpecification.java From flink with Apache License 2.0 | 5 votes |
public static ClusterSpecification fromConfiguration(Configuration configuration) { int slots = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS, 1); int jobManagerMemoryMb = ConfigurationUtils.getJobManagerHeapMemory(configuration).getMebiBytes(); int taskManagerMemoryMb = ConfigurationUtils.getTaskManagerHeapMemory(configuration).getMebiBytes(); return new ClusterSpecificationBuilder() .setMasterMemoryMB(jobManagerMemoryMb) .setTaskManagerMemoryMB(taskManagerMemoryMb) .setNumberTaskManagers(1) .setSlotsPerTaskManager(slots) .createClusterSpecification(); }
Example #30
Source File: BashJavaUtilsITCase.java From flink with Apache License 2.0 | 5 votes |
@Test public void testGetTmResourceParamsConfigs() throws Exception { int expectedResultLines = 2; String[] commands = {RUN_BASH_JAVA_UTILS_CMD_SCRIPT, BashJavaUtils.Command.GET_TM_RESOURCE_PARAMS.toString(), String.valueOf(expectedResultLines)}; List<String> lines = Arrays.asList(executeScript(commands).split(System.lineSeparator())); assertThat(lines.size(), is(expectedResultLines)); ConfigurationUtils.parseJvmArgString(lines.get(0)); ConfigurationUtils.parseTmResourceDynamicConfigs(lines.get(1)); }