Java Code Examples for org.apache.flink.client.deployment.ClusterSpecification#getTaskManagerMemoryMB()

The following examples show how to use org.apache.flink.client.deployment.ClusterSpecification#getTaskManagerMemoryMB() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: AbstractYarnClusterDescriptor.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
/**
 * Method to validate cluster specification before deploy it, it will throw
 * an {@link FlinkException} if the {@link ClusterSpecification} is invalid.
 *
 * @param clusterSpecification cluster specification to check against the configuration of the
 *                             AbstractYarnClusterDescriptor
 * @throws FlinkException if the cluster cannot be started with the provided {@link ClusterSpecification}
 */
private void validateClusterSpecification(ClusterSpecification clusterSpecification) throws FlinkException {
	try {
		final long taskManagerMemorySize = clusterSpecification.getTaskManagerMemoryMB();
		// We do the validation by calling the calculation methods here
		// Internally these methods will check whether the cluster can be started with the provided
		// ClusterSpecification and the configured memory requirements
		final long cutoff = ContaineredTaskManagerParameters.calculateCutoffMB(flinkConfiguration, taskManagerMemorySize);
		TaskManagerServices.calculateHeapSizeMB(taskManagerMemorySize - cutoff, flinkConfiguration);
	} catch (IllegalArgumentException iae) {
		throw new FlinkException("Cannot fulfill the minimum memory requirements with the provided " +
			"cluster specification. Please increase the memory of the cluster.", iae);
	}
}
Example 2
Source File: AbstractYarnClusterDescriptor.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Method to validate cluster specification before deploy it, it will throw
 * an {@link FlinkException} if the {@link ClusterSpecification} is invalid.
 *
 * @param clusterSpecification cluster specification to check against the configuration of the
 *                             AbstractYarnClusterDescriptor
 * @throws FlinkException if the cluster cannot be started with the provided {@link ClusterSpecification}
 */
private void validateClusterSpecification(ClusterSpecification clusterSpecification) throws FlinkException {
	try {
		final long taskManagerMemorySize = clusterSpecification.getTaskManagerMemoryMB();
		// We do the validation by calling the calculation methods here
		// Internally these methods will check whether the cluster can be started with the provided
		// ClusterSpecification and the configured memory requirements
		final long cutoff = ContaineredTaskManagerParameters.calculateCutoffMB(flinkConfiguration, taskManagerMemorySize);
		TaskManagerServices.calculateHeapSizeMB(taskManagerMemorySize - cutoff, flinkConfiguration);
	} catch (IllegalArgumentException iae) {
		throw new FlinkException("Cannot fulfill the minimum memory requirements with the provided " +
			"cluster specification. Please increase the memory of the cluster.", iae);
	}
}
Example 3
Source File: AbstractYarnClusterDescriptor.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
protected ClusterSpecification validateClusterResources(
	ClusterSpecification clusterSpecification,
	int yarnMinAllocationMB,
	Resource maximumResourceCapability,
	ClusterResourceDescription freeClusterResources) throws YarnDeploymentException {

	int taskManagerCount = clusterSpecification.getNumberTaskManagers();
	int jobManagerMemoryMb = clusterSpecification.getMasterMemoryMB();
	int taskManagerMemoryMb = clusterSpecification.getTaskManagerMemoryMB();

	if (jobManagerMemoryMb < yarnMinAllocationMB || taskManagerMemoryMb < yarnMinAllocationMB) {
		LOG.warn("The JobManager or TaskManager memory is below the smallest possible YARN Container size. "
			+ "The value of 'yarn.scheduler.minimum-allocation-mb' is '" + yarnMinAllocationMB + "'. Please increase the memory size." +
			"YARN will allocate the smaller containers but the scheduler will account for the minimum-allocation-mb, maybe not all instances " +
			"you requested will start.");
	}

	// set the memory to minAllocationMB to do the next checks correctly
	if (jobManagerMemoryMb < yarnMinAllocationMB) {
		jobManagerMemoryMb =  yarnMinAllocationMB;
	}
	if (taskManagerMemoryMb < yarnMinAllocationMB) {
		taskManagerMemoryMb =  yarnMinAllocationMB;
	}

	final String note = "Please check the 'yarn.scheduler.maximum-allocation-mb' and the 'yarn.nodemanager.resource.memory-mb' configuration values\n";
	if (jobManagerMemoryMb > maximumResourceCapability.getMemory()) {
		throw new YarnDeploymentException("The cluster does not have the requested resources for the JobManager available!\n"
			+ "Maximum Memory: " + maximumResourceCapability.getMemory() + "MB Requested: " + jobManagerMemoryMb + "MB. " + note);
	}

	if (taskManagerMemoryMb > maximumResourceCapability.getMemory()) {
		throw new YarnDeploymentException("The cluster does not have the requested resources for the TaskManagers available!\n"
			+ "Maximum Memory: " + maximumResourceCapability.getMemory() + " Requested: " + taskManagerMemoryMb + "MB. " + note);
	}

	final String noteRsc = "\nThe Flink YARN client will try to allocate the YARN session, but maybe not all TaskManagers are " +
		"connecting from the beginning because the resources are currently not available in the cluster. " +
		"The allocation might take more time than usual because the Flink YARN client needs to wait until " +
		"the resources become available.";
	int totalMemoryRequired = jobManagerMemoryMb + taskManagerMemoryMb * taskManagerCount;

	if (freeClusterResources.totalFreeMemory < totalMemoryRequired) {
		LOG.warn("This YARN session requires " + totalMemoryRequired + "MB of memory in the cluster. "
			+ "There are currently only " + freeClusterResources.totalFreeMemory + "MB available." + noteRsc);

	}
	if (taskManagerMemoryMb > freeClusterResources.containerLimit) {
		LOG.warn("The requested amount of memory for the TaskManagers (" + taskManagerMemoryMb + "MB) is more than "
			+ "the largest possible YARN container: " + freeClusterResources.containerLimit + noteRsc);
	}
	if (jobManagerMemoryMb > freeClusterResources.containerLimit) {
		LOG.warn("The requested amount of memory for the JobManager (" + jobManagerMemoryMb + "MB) is more than "
			+ "the largest possible YARN container: " + freeClusterResources.containerLimit + noteRsc);
	}

	// ----------------- check if the requested containers fit into the cluster.

	int[] nmFree = Arrays.copyOf(freeClusterResources.nodeManagersFree, freeClusterResources.nodeManagersFree.length);
	// first, allocate the jobManager somewhere.
	if (!allocateResource(nmFree, jobManagerMemoryMb)) {
		LOG.warn("Unable to find a NodeManager that can fit the JobManager/Application master. " +
			"The JobManager requires " + jobManagerMemoryMb + "MB. NodeManagers available: " +
			Arrays.toString(freeClusterResources.nodeManagersFree) + noteRsc);
	}
	// allocate TaskManagers
	for (int i = 0; i < taskManagerCount; i++) {
		if (!allocateResource(nmFree, taskManagerMemoryMb)) {
			LOG.warn("There is not enough memory available in the YARN cluster. " +
				"The TaskManager(s) require " + taskManagerMemoryMb + "MB each. " +
				"NodeManagers available: " + Arrays.toString(freeClusterResources.nodeManagersFree) + "\n" +
				"After allocating the JobManager (" + jobManagerMemoryMb + "MB) and (" + i + "/" + taskManagerCount + ") TaskManagers, " +
				"the following NodeManagers are available: " + Arrays.toString(nmFree)  + noteRsc);
		}
	}

	return new ClusterSpecification.ClusterSpecificationBuilder()
		.setMasterMemoryMB(jobManagerMemoryMb)
		.setTaskManagerMemoryMB(taskManagerMemoryMb)
		.setNumberTaskManagers(clusterSpecification.getNumberTaskManagers())
		.setSlotsPerTaskManager(clusterSpecification.getSlotsPerTaskManager())
		.createClusterSpecification();

}
Example 4
Source File: AbstractYarnClusterDescriptor.java    From flink with Apache License 2.0 4 votes vote down vote up 
protected ClusterSpecification validateClusterResources(
	ClusterSpecification clusterSpecification,
	int yarnMinAllocationMB,
	Resource maximumResourceCapability,
	ClusterResourceDescription freeClusterResources) throws YarnDeploymentException {

	int taskManagerCount = clusterSpecification.getNumberTaskManagers();
	int jobManagerMemoryMb = clusterSpecification.getMasterMemoryMB();
	int taskManagerMemoryMb = clusterSpecification.getTaskManagerMemoryMB();

	if (jobManagerMemoryMb < yarnMinAllocationMB || taskManagerMemoryMb < yarnMinAllocationMB) {
		LOG.warn("The JobManager or TaskManager memory is below the smallest possible YARN Container size. "
			+ "The value of 'yarn.scheduler.minimum-allocation-mb' is '" + yarnMinAllocationMB + "'. Please increase the memory size." +
			"YARN will allocate the smaller containers but the scheduler will account for the minimum-allocation-mb, maybe not all instances " +
			"you requested will start.");
	}

	// set the memory to minAllocationMB to do the next checks correctly
	if (jobManagerMemoryMb < yarnMinAllocationMB) {
		jobManagerMemoryMb =  yarnMinAllocationMB;
	}
	if (taskManagerMemoryMb < yarnMinAllocationMB) {
		taskManagerMemoryMb =  yarnMinAllocationMB;
	}

	final String note = "Please check the 'yarn.scheduler.maximum-allocation-mb' and the 'yarn.nodemanager.resource.memory-mb' configuration values\n";
	if (jobManagerMemoryMb > maximumResourceCapability.getMemory()) {
		throw new YarnDeploymentException("The cluster does not have the requested resources for the JobManager available!\n"
			+ "Maximum Memory: " + maximumResourceCapability.getMemory() + "MB Requested: " + jobManagerMemoryMb + "MB. " + note);
	}

	if (taskManagerMemoryMb > maximumResourceCapability.getMemory()) {
		throw new YarnDeploymentException("The cluster does not have the requested resources for the TaskManagers available!\n"
			+ "Maximum Memory: " + maximumResourceCapability.getMemory() + " Requested: " + taskManagerMemoryMb + "MB. " + note);
	}

	final String noteRsc = "\nThe Flink YARN client will try to allocate the YARN session, but maybe not all TaskManagers are " +
		"connecting from the beginning because the resources are currently not available in the cluster. " +
		"The allocation might take more time than usual because the Flink YARN client needs to wait until " +
		"the resources become available.";
	int totalMemoryRequired = jobManagerMemoryMb + taskManagerMemoryMb * taskManagerCount;

	if (freeClusterResources.totalFreeMemory < totalMemoryRequired) {
		LOG.warn("This YARN session requires " + totalMemoryRequired + "MB of memory in the cluster. "
			+ "There are currently only " + freeClusterResources.totalFreeMemory + "MB available." + noteRsc);

	}
	if (taskManagerMemoryMb > freeClusterResources.containerLimit) {
		LOG.warn("The requested amount of memory for the TaskManagers (" + taskManagerMemoryMb + "MB) is more than "
			+ "the largest possible YARN container: " + freeClusterResources.containerLimit + noteRsc);
	}
	if (jobManagerMemoryMb > freeClusterResources.containerLimit) {
		LOG.warn("The requested amount of memory for the JobManager (" + jobManagerMemoryMb + "MB) is more than "
			+ "the largest possible YARN container: " + freeClusterResources.containerLimit + noteRsc);
	}

	// ----------------- check if the requested containers fit into the cluster.

	int[] nmFree = Arrays.copyOf(freeClusterResources.nodeManagersFree, freeClusterResources.nodeManagersFree.length);
	// first, allocate the jobManager somewhere.
	if (!allocateResource(nmFree, jobManagerMemoryMb)) {
		LOG.warn("Unable to find a NodeManager that can fit the JobManager/Application master. " +
			"The JobManager requires " + jobManagerMemoryMb + "MB. NodeManagers available: " +
			Arrays.toString(freeClusterResources.nodeManagersFree) + noteRsc);
	}
	// allocate TaskManagers
	for (int i = 0; i < taskManagerCount; i++) {
		if (!allocateResource(nmFree, taskManagerMemoryMb)) {
			LOG.warn("There is not enough memory available in the YARN cluster. " +
				"The TaskManager(s) require " + taskManagerMemoryMb + "MB each. " +
				"NodeManagers available: " + Arrays.toString(freeClusterResources.nodeManagersFree) + "\n" +
				"After allocating the JobManager (" + jobManagerMemoryMb + "MB) and (" + i + "/" + taskManagerCount + ") TaskManagers, " +
				"the following NodeManagers are available: " + Arrays.toString(nmFree)  + noteRsc);
		}
	}

	return new ClusterSpecification.ClusterSpecificationBuilder()
		.setMasterMemoryMB(jobManagerMemoryMb)
		.setTaskManagerMemoryMB(taskManagerMemoryMb)
		.setNumberTaskManagers(clusterSpecification.getNumberTaskManagers())
		.setSlotsPerTaskManager(clusterSpecification.getSlotsPerTaskManager())
		.createClusterSpecification();

}
Example 5
Source File: YarnClusterDescriptor.java    From flink with Apache License 2.0 4 votes vote down vote up 
private ClusterSpecification validateClusterResources(
	ClusterSpecification clusterSpecification,
	int yarnMinAllocationMB,
	Resource maximumResourceCapability,
	ClusterResourceDescription freeClusterResources) throws YarnDeploymentException {

	int jobManagerMemoryMb = clusterSpecification.getMasterMemoryMB();
	final int taskManagerMemoryMb = clusterSpecification.getTaskManagerMemoryMB();

	logIfComponentMemNotIntegerMultipleOfYarnMinAllocation("JobManager", jobManagerMemoryMb, yarnMinAllocationMB);
	logIfComponentMemNotIntegerMultipleOfYarnMinAllocation("TaskManager", taskManagerMemoryMb, yarnMinAllocationMB);

	// set the memory to minAllocationMB to do the next checks correctly
	if (jobManagerMemoryMb < yarnMinAllocationMB) {
		jobManagerMemoryMb =  yarnMinAllocationMB;
	}

	final String note = "Please check the 'yarn.scheduler.maximum-allocation-mb' and the 'yarn.nodemanager.resource.memory-mb' configuration values\n";
	if (jobManagerMemoryMb > maximumResourceCapability.getMemory()) {
		throw new YarnDeploymentException("The cluster does not have the requested resources for the JobManager available!\n"
				+ "Maximum Memory: " + maximumResourceCapability.getMemory() + "MB Requested: " + jobManagerMemoryMb + "MB. " + note);
	}

	if (taskManagerMemoryMb > maximumResourceCapability.getMemory()) {
		throw new YarnDeploymentException("The cluster does not have the requested resources for the TaskManagers available!\n"
				+ "Maximum Memory: " + maximumResourceCapability.getMemory() + " Requested: " + taskManagerMemoryMb + "MB. " + note);
	}

	final String noteRsc = "\nThe Flink YARN client will try to allocate the YARN session, but maybe not all TaskManagers are " +
			"connecting from the beginning because the resources are currently not available in the cluster. " +
			"The allocation might take more time than usual because the Flink YARN client needs to wait until " +
			"the resources become available.";

	if (taskManagerMemoryMb > freeClusterResources.containerLimit) {
		LOG.warn("The requested amount of memory for the TaskManagers (" + taskManagerMemoryMb + "MB) is more than "
				+ "the largest possible YARN container: " + freeClusterResources.containerLimit + noteRsc);
	}
	if (jobManagerMemoryMb > freeClusterResources.containerLimit) {
		LOG.warn("The requested amount of memory for the JobManager (" + jobManagerMemoryMb + "MB) is more than "
				+ "the largest possible YARN container: " + freeClusterResources.containerLimit + noteRsc);
	}

	return new ClusterSpecification.ClusterSpecificationBuilder()
			.setMasterMemoryMB(jobManagerMemoryMb)
			.setTaskManagerMemoryMB(taskManagerMemoryMb)
			.setSlotsPerTaskManager(clusterSpecification.getSlotsPerTaskManager())
			.createClusterSpecification();

}