Java Code Examples for org.apache.mesos.Protos#Parameter

The following examples show how to use org.apache.mesos.Protos#Parameter . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: MesosTaskManagerParameters.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
public static List<Protos.Parameter> buildDockerParameters(Option<String> dockerParameters) {
	if (dockerParameters.isEmpty()) {
		return Collections.emptyList();
	} else {
		String[] dockerParameterSpecifications = dockerParameters.get().split(",");

		List<Protos.Parameter> parameters = new ArrayList<>(dockerParameterSpecifications.length);

		for (String dockerParameterSpecification : dockerParameterSpecifications) {
			if (!dockerParameterSpecification.trim().isEmpty()) {
				// split with the limit of 2 in case the value includes '='
				String[] match = dockerParameterSpecification.split("=", 2);
				if (match.length != 2) {
					throw new IllegalArgumentException("Docker parameter specification is invalid, given: "
						+ dockerParameterSpecification);
				}
				Protos.Parameter.Builder parameter = Protos.Parameter.newBuilder();
				parameter.setKey(match[0]);
				parameter.setValue(match[1]);
				parameters.add(parameter.build());
			}
		}
		return parameters;
	}
}
Example 2
Source File: MesosTaskManagerParameters.java    From flink with Apache License 2.0 6 votes vote down vote up 
public static List<Protos.Parameter> buildDockerParameters(Option<String> dockerParameters) {
	if (dockerParameters.isEmpty()) {
		return Collections.emptyList();
	} else {
		String[] dockerParameterSpecifications = dockerParameters.get().split(",");

		List<Protos.Parameter> parameters = new ArrayList<>(dockerParameterSpecifications.length);

		for (String dockerParameterSpecification : dockerParameterSpecifications) {
			if (!dockerParameterSpecification.trim().isEmpty()) {
				// split with the limit of 2 in case the value includes '='
				String[] match = dockerParameterSpecification.split("=", 2);
				if (match.length != 2) {
					throw new IllegalArgumentException("Docker parameter specification is invalid, given: "
						+ dockerParameterSpecification);
				}
				Protos.Parameter.Builder parameter = Protos.Parameter.newBuilder();
				parameter.setKey(match[0]);
				parameter.setValue(match[1]);
				parameters.add(parameter.build());
			}
		}
		return parameters;
	}
}
Example 3
Source File: MesosTaskManagerParameters.java    From flink with Apache License 2.0 6 votes vote down vote up 
public static List<Protos.Parameter> buildDockerParameters(Option<String> dockerParameters) {
	if (dockerParameters.isEmpty()) {
		return Collections.emptyList();
	} else {
		String[] dockerParameterSpecifications = dockerParameters.get().split(",");

		List<Protos.Parameter> parameters = new ArrayList<>(dockerParameterSpecifications.length);

		for (String dockerParameterSpecification : dockerParameterSpecifications) {
			if (!dockerParameterSpecification.trim().isEmpty()) {
				// split with the limit of 2 in case the value includes '='
				String[] match = dockerParameterSpecification.split("=", 2);
				if (match.length != 2) {
					throw new IllegalArgumentException("Docker parameter specification is invalid, given: "
						+ dockerParameterSpecification);
				}
				Protos.Parameter.Builder parameter = Protos.Parameter.newBuilder();
				parameter.setKey(match[0]);
				parameter.setValue(match[1]);
				parameters.add(parameter.build());
			}
		}
		return parameters;
	}
}
Example 4
Source File: MesosTaskManagerParameters.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
public MesosTaskManagerParameters(
		double cpus,
		int gpus,
		ContainerType containerType,
		Option<String> containerImageName,
		ContaineredTaskManagerParameters containeredParameters,
		List<Protos.Volume> containerVolumes,
		List<Protos.Parameter> dockerParameters,
		boolean dockerForcePullImage,
		List<ConstraintEvaluator> constraints,
		String command,
		Option<String> bootstrapCommand,
		Option<String> taskManagerHostname,
		List<String> uris) {

	this.cpus = cpus;
	this.gpus = gpus;
	this.containerType = Preconditions.checkNotNull(containerType);
	this.containerImageName = Preconditions.checkNotNull(containerImageName);
	this.containeredParameters = Preconditions.checkNotNull(containeredParameters);
	this.containerVolumes = Preconditions.checkNotNull(containerVolumes);
	this.dockerParameters = Preconditions.checkNotNull(dockerParameters);
	this.dockerForcePullImage = dockerForcePullImage;
	this.constraints = Preconditions.checkNotNull(constraints);
	this.command = Preconditions.checkNotNull(command);
	this.bootstrapCommand = Preconditions.checkNotNull(bootstrapCommand);
	this.taskManagerHostname = Preconditions.checkNotNull(taskManagerHostname);
	this.uris = Preconditions.checkNotNull(uris);
}
Example 5
Source File: MesosTaskManagerParameters.java    From flink with Apache License 2.0 5 votes vote down vote up 
public MesosTaskManagerParameters(
		double cpus,
		int gpus,
		int disk,
		ContainerType containerType,
		Option<String> containerImageName,
		ContaineredTaskManagerParameters containeredParameters,
		List<Protos.Volume> containerVolumes,
		List<Protos.Parameter> dockerParameters,
		boolean dockerForcePullImage,
		List<ConstraintEvaluator> constraints,
		String command,
		Option<String> bootstrapCommand,
		Option<String> taskManagerHostname,
		List<String> uris) {

	this.cpus = cpus;
	this.gpus = gpus;
	this.disk = disk;
	this.containerType = Preconditions.checkNotNull(containerType);
	this.containerImageName = Preconditions.checkNotNull(containerImageName);
	this.containeredParameters = Preconditions.checkNotNull(containeredParameters);
	this.containerVolumes = Preconditions.checkNotNull(containerVolumes);
	this.dockerParameters = Preconditions.checkNotNull(dockerParameters);
	this.dockerForcePullImage = dockerForcePullImage;
	this.constraints = Preconditions.checkNotNull(constraints);
	this.command = Preconditions.checkNotNull(command);
	this.bootstrapCommand = Preconditions.checkNotNull(bootstrapCommand);
	this.taskManagerHostname = Preconditions.checkNotNull(taskManagerHostname);
	this.uris = Preconditions.checkNotNull(uris);
}
Example 6
Source File: TaskFactory.java    From incubator-myriad with Apache License 2.0 5 votes vote down vote up 
protected Iterable<Protos.Parameter> getParameters(Iterable<Map<String, String>> params) {
  Preconditions.checkNotNull(params);
  return Iterables.transform(params, new Function<Map<String, String>, Protos.Parameter>() {
    @Override
    public Protos.Parameter apply(Map<String, String> parameter) {
      Preconditions.checkNotNull(parameter, "Null parameter");
      Preconditions.checkState(parameter.containsKey(PARAMETER_KEY_KEY), "Missing key");
      Preconditions.checkState(parameter.containsKey(PARAMETER_VALUE_KEY), "Missing value");
      return Protos.Parameter.newBuilder()
          .setKey(parameter.get(PARAMETER_KEY_KEY))
          .setValue(PARAMETER_VALUE_KEY)
          .build();
    }
  });
}
Example 7
Source File: MesosTaskManagerParameters.java    From flink with Apache License 2.0 5 votes vote down vote up 
public MesosTaskManagerParameters(
		int gpus,
		int disk,
		int network,
		ContainerType containerType,
		Option<String> containerImageName,
		ContaineredTaskManagerParameters containeredParameters,
		List<Protos.Volume> containerVolumes,
		List<Protos.Parameter> dockerParameters,
		boolean dockerForcePullImage,
		List<ConstraintEvaluator> constraints,
		String command,
		Option<String> bootstrapCommand,
		Option<String> taskManagerHostname,
		List<String> uris) {

	this.gpus = gpus;
	this.disk = disk;
	this.network = network;
	this.containerType = Preconditions.checkNotNull(containerType);
	this.containerImageName = Preconditions.checkNotNull(containerImageName);
	this.containeredParameters = Preconditions.checkNotNull(containeredParameters);
	this.containerVolumes = Preconditions.checkNotNull(containerVolumes);
	this.dockerParameters = Preconditions.checkNotNull(dockerParameters);
	this.dockerForcePullImage = dockerForcePullImage;
	this.constraints = Preconditions.checkNotNull(constraints);
	this.command = Preconditions.checkNotNull(command);
	this.bootstrapCommand = Preconditions.checkNotNull(bootstrapCommand);
	this.taskManagerHostname = Preconditions.checkNotNull(taskManagerHostname);
	this.uris = Preconditions.checkNotNull(uris);
}
Example 8
Source File: MesosTaskManagerParameters.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
/**
 * Get Docker runtime parameters.
 */
public List<Protos.Parameter> dockerParameters() {
	return dockerParameters;
}
Example 9
Source File: MesosTaskManagerParameters.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
/**
 * Create the Mesos TaskManager parameters.
 *
 * @param flinkConfig the TM configuration.
 */
public static MesosTaskManagerParameters create(Configuration flinkConfig) {

	List<ConstraintEvaluator> constraints = parseConstraints(flinkConfig.getString(MESOS_CONSTRAINTS_HARD_HOSTATTR));
	// parse the common parameters
	ContaineredTaskManagerParameters containeredParameters = ContaineredTaskManagerParameters.create(
		flinkConfig,
		flinkConfig.getInteger(MESOS_RM_TASKS_MEMORY_MB),
		flinkConfig.getInteger(MESOS_RM_TASKS_SLOTS));

	double cpus = flinkConfig.getDouble(MESOS_RM_TASKS_CPUS);
	if (cpus <= 0.0) {
		cpus = Math.max(containeredParameters.numSlots(), 1.0);
	}

	int gpus = flinkConfig.getInteger(MESOS_RM_TASKS_GPUS);

	if (gpus < 0) {
		throw new IllegalConfigurationException(MESOS_RM_TASKS_GPUS.key() +
			" cannot be negative");
	}

	// parse the containerization parameters
	String imageName = flinkConfig.getString(MESOS_RM_CONTAINER_IMAGE_NAME);

	ContainerType containerType;
	String containerTypeString = flinkConfig.getString(MESOS_RM_CONTAINER_TYPE);
	switch (containerTypeString) {
		case MESOS_RESOURCEMANAGER_TASKS_CONTAINER_TYPE_MESOS:
			containerType = ContainerType.MESOS;
			break;
		case MESOS_RESOURCEMANAGER_TASKS_CONTAINER_TYPE_DOCKER:
			containerType = ContainerType.DOCKER;
			if (imageName == null || imageName.length() == 0) {
				throw new IllegalConfigurationException(MESOS_RM_CONTAINER_IMAGE_NAME.key() +
					" must be specified for docker container type");
			}
			break;
		default:
			throw new IllegalConfigurationException("invalid container type: " + containerTypeString);
	}

	Option<String> containerVolOpt = Option.<String>apply(flinkConfig.getString(MESOS_RM_CONTAINER_VOLUMES));

	Option<String> dockerParamsOpt = Option.<String>apply(flinkConfig.getString(MESOS_RM_CONTAINER_DOCKER_PARAMETERS));

	Option<String> uriParamsOpt = Option.<String>apply(flinkConfig.getString(MESOS_TM_URIS));

	boolean dockerForcePullImage = flinkConfig.getBoolean(MESOS_RM_CONTAINER_DOCKER_FORCE_PULL_IMAGE);

	List<Protos.Volume> containerVolumes = buildVolumes(containerVolOpt);

	List<Protos.Parameter> dockerParameters = buildDockerParameters(dockerParamsOpt);

	List<String> uris = buildUris(uriParamsOpt);

	//obtain Task Manager Host Name from the configuration
	Option<String> taskManagerHostname = Option.apply(flinkConfig.getString(MESOS_TM_HOSTNAME));

	//obtain command-line from the configuration
	String tmCommand = flinkConfig.getString(MESOS_TM_CMD);
	Option<String> tmBootstrapCommand = Option.apply(flinkConfig.getString(MESOS_TM_BOOTSTRAP_CMD));

	return new MesosTaskManagerParameters(
		cpus,
		gpus,
		containerType,
		Option.apply(imageName),
		containeredParameters,
		containerVolumes,
		dockerParameters,
		dockerForcePullImage,
		constraints,
		tmCommand,
		tmBootstrapCommand,
		taskManagerHostname,
		uris);
}
Example 10
Source File: MesosTaskManagerParameters.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * Get Docker runtime parameters.
 */
public List<Protos.Parameter> dockerParameters() {
	return dockerParameters;
}
Example 11
Source File: MesosTaskManagerParameters.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * Create the Mesos TaskManager parameters.
 *
 * @param flinkConfig the TM configuration.
 */
public static MesosTaskManagerParameters create(Configuration flinkConfig) {

	List<ConstraintEvaluator> constraints = parseConstraints(flinkConfig.getString(MESOS_CONSTRAINTS_HARD_HOSTATTR));
	// parse the common parameters
	ContaineredTaskManagerParameters containeredParameters = ContaineredTaskManagerParameters.create(
		flinkConfig,
		flinkConfig.getInteger(MESOS_RM_TASKS_MEMORY_MB),
		flinkConfig.getInteger(MESOS_RM_TASKS_SLOTS));

	double cpus = flinkConfig.getDouble(MESOS_RM_TASKS_CPUS);
	if (cpus <= 0.0) {
		cpus = Math.max(containeredParameters.numSlots(), 1.0);
	}

	int gpus = flinkConfig.getInteger(MESOS_RM_TASKS_GPUS);

	if (gpus < 0) {
		throw new IllegalConfigurationException(MESOS_RM_TASKS_GPUS.key() +
			" cannot be negative");
	}

	int disk = flinkConfig.getInteger(MESOS_RM_TASKS_DISK_MB);

	// parse the containerization parameters
	String imageName = flinkConfig.getString(MESOS_RM_CONTAINER_IMAGE_NAME);

	ContainerType containerType;
	String containerTypeString = flinkConfig.getString(MESOS_RM_CONTAINER_TYPE);
	switch (containerTypeString) {
		case MESOS_RESOURCEMANAGER_TASKS_CONTAINER_TYPE_MESOS:
			containerType = ContainerType.MESOS;
			break;
		case MESOS_RESOURCEMANAGER_TASKS_CONTAINER_TYPE_DOCKER:
			containerType = ContainerType.DOCKER;
			if (imageName == null || imageName.length() == 0) {
				throw new IllegalConfigurationException(MESOS_RM_CONTAINER_IMAGE_NAME.key() +
					" must be specified for docker container type");
			}
			break;
		default:
			throw new IllegalConfigurationException("invalid container type: " + containerTypeString);
	}

	Option<String> containerVolOpt = Option.<String>apply(flinkConfig.getString(MESOS_RM_CONTAINER_VOLUMES));

	Option<String> dockerParamsOpt = Option.<String>apply(flinkConfig.getString(MESOS_RM_CONTAINER_DOCKER_PARAMETERS));

	Option<String> uriParamsOpt = Option.<String>apply(flinkConfig.getString(MESOS_TM_URIS));

	boolean dockerForcePullImage = flinkConfig.getBoolean(MESOS_RM_CONTAINER_DOCKER_FORCE_PULL_IMAGE);

	List<Protos.Volume> containerVolumes = buildVolumes(containerVolOpt);

	List<Protos.Parameter> dockerParameters = buildDockerParameters(dockerParamsOpt);

	List<String> uris = buildUris(uriParamsOpt);

	//obtain Task Manager Host Name from the configuration
	Option<String> taskManagerHostname = Option.apply(flinkConfig.getString(MESOS_TM_HOSTNAME));

	//obtain command-line from the configuration
	String tmCommand = flinkConfig.getString(MESOS_TM_CMD);
	Option<String> tmBootstrapCommand = Option.apply(flinkConfig.getString(MESOS_TM_BOOTSTRAP_CMD));

	return new MesosTaskManagerParameters(
		cpus,
		gpus,
		disk,
		containerType,
		Option.apply(imageName),
		containeredParameters,
		containerVolumes,
		dockerParameters,
		dockerForcePullImage,
		constraints,
		tmCommand,
		tmBootstrapCommand,
		taskManagerHostname,
		uris);
}
Example 12
Source File: MesosTaskManagerParameters.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * Get Docker runtime parameters.
 */
public List<Protos.Parameter> dockerParameters() {
	return dockerParameters;
}
Example 13
Source File: MesosTaskManagerParameters.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * Create the Mesos TaskManager parameters.
 *
 * @param flinkConfig the TM configuration.
 */
public static MesosTaskManagerParameters create(Configuration flinkConfig) {

	List<ConstraintEvaluator> constraints = parseConstraints(flinkConfig.getString(MESOS_CONSTRAINTS_HARD_HOSTATTR));

	// parse the common parameters
	ContaineredTaskManagerParameters containeredParameters = createContaineredTaskManagerParameters(flinkConfig);

	int gpus = flinkConfig.getInteger(MESOS_RM_TASKS_GPUS);

	if (gpus < 0) {
		throw new IllegalConfigurationException(MESOS_RM_TASKS_GPUS.key() +
			" cannot be negative");
	}

	int disk = flinkConfig.getInteger(MESOS_RM_TASKS_DISK_MB);

	int network = flinkConfig.getInteger(MESOS_RM_TASKS_NETWORK_MB_PER_SEC);

	// parse the containerization parameters
	String imageName = flinkConfig.getString(MESOS_RM_CONTAINER_IMAGE_NAME);

	ContainerType containerType;
	String containerTypeString = flinkConfig.getString(MESOS_RM_CONTAINER_TYPE);
	switch (containerTypeString) {
		case MESOS_RESOURCEMANAGER_TASKS_CONTAINER_TYPE_MESOS:
			containerType = ContainerType.MESOS;
			break;
		case MESOS_RESOURCEMANAGER_TASKS_CONTAINER_TYPE_DOCKER:
			containerType = ContainerType.DOCKER;
			if (imageName == null || imageName.length() == 0) {
				throw new IllegalConfigurationException(MESOS_RM_CONTAINER_IMAGE_NAME.key() +
					" must be specified for docker container type");
			}
			break;
		default:
			throw new IllegalConfigurationException("invalid container type: " + containerTypeString);
	}

	Option<String> containerVolOpt = Option.<String>apply(flinkConfig.getString(MESOS_RM_CONTAINER_VOLUMES));

	Option<String> dockerParamsOpt = Option.<String>apply(flinkConfig.getString(MESOS_RM_CONTAINER_DOCKER_PARAMETERS));

	Option<String> uriParamsOpt = Option.<String>apply(flinkConfig.getString(MESOS_TM_URIS));

	boolean dockerForcePullImage = flinkConfig.getBoolean(MESOS_RM_CONTAINER_DOCKER_FORCE_PULL_IMAGE);

	List<Protos.Volume> containerVolumes = buildVolumes(containerVolOpt);

	List<Protos.Parameter> dockerParameters = buildDockerParameters(dockerParamsOpt);

	List<String> uris = buildUris(uriParamsOpt);

	//obtain Task Manager Host Name from the configuration
	Option<String> taskManagerHostname = Option.apply(flinkConfig.getString(MESOS_TM_HOSTNAME));

	//obtain command-line from the configuration
	String tmCommand = flinkConfig.getString(MESOS_TM_CMD);
	Option<String> tmBootstrapCommand = Option.apply(flinkConfig.getString(MESOS_TM_BOOTSTRAP_CMD));

	return new MesosTaskManagerParameters(
		gpus,
		disk,
		network,
		containerType,
		Option.apply(imageName),
		containeredParameters,
		containerVolumes,
		dockerParameters,
		dockerForcePullImage,
		constraints,
		tmCommand,
		tmBootstrapCommand,
		taskManagerHostname,
		uris);
}