Java Code Examples for org.apache.mesos.Protos#Environment

The following examples show how to use org.apache.mesos.Protos#Environment . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: EnvUtilsTest.java    From dcos-commons with Apache License 2.0 6 votes vote down vote up 
@Test
public void addEnvVarToEnvironmentWithSecret() {
    Protos.Environment.Builder envBuilder = Protos.Environment.newBuilder();
    envBuilder.addVariablesBuilder()
            .setName(SECRET_KEY)
            .setType(Protos.Environment.Variable.Type.SECRET)
            .setSecret(getReferenceSecret(SECRET_PATH));
    Protos.Environment newEnv = EnvUtils.withEnvVar(envBuilder.build(), TEST_ENV_KEY, TEST_ENV_VALUE);
    Assert.assertEquals(newEnv.getVariablesCount(), 2);
    for(Protos.Environment.Variable envVar : newEnv.getVariablesList()) {
        if (envVar.getName().equals(SECRET_KEY)) {
            Assert.assertEquals(envVar.getSecret(), getReferenceSecret(SECRET_PATH));
        }
        if (envVar.getName().equals(TEST_ENV_KEY)) {
            Assert.assertEquals(envVar.getValue(), TEST_ENV_VALUE);
        }

    }
}
Example 2
Source File: TaskInfoFactory.java    From elasticsearch with Apache License 2.0 6 votes vote down vote up 
private Protos.CommandInfo nativeCommand(Configuration configuration, List<String> args, Long elasticSearchNodeId) {
    String address = configuration.getFrameworkFileServerAddress();
    if (address == null) {
        throw new NullPointerException("Webserver address is null");
    }
    String httpPath = address + "/get/" + Configuration.ES_TAR;
    String command = configuration.nativeCommand(args);
    final Protos.Environment environment = Protos.Environment.newBuilder().addAllVariables(new ExecutorEnvironmentalVariables(configuration, elasticSearchNodeId).getList()).build();
    final Protos.CommandInfo.Builder builder = Protos.CommandInfo.newBuilder()
            .setShell(true)
            .setValue(command)
            .setUser("root")
            .mergeEnvironment(environment);
    if (configuration.getElasticsearchBinary().isEmpty()) {
        builder.addUris(Protos.CommandInfo.URI.newBuilder().setValue(httpPath));
    } else {
        builder.addUris(Protos.CommandInfo.URI.newBuilder().setValue(configuration.getElasticsearchBinary()));
    }
    if (!configuration.getElasticsearchSettingsLocation().isEmpty()) {
        builder.addUris(Protos.CommandInfo.URI.newBuilder().setValue(configuration.getElasticsearchSettingsLocation()));
    }
    return builder
            .build();
}
Example 3
Source File: EnvUtilsTest.java    From dcos-commons with Apache License 2.0 5 votes vote down vote up 
@Test
public void overwriteExistingEnvVarKey() {
    Protos.Environment.Builder envBuilder = Protos.Environment.newBuilder();
    envBuilder.addVariablesBuilder()
            .setName(TEST_ENV_KEY)
            .setValue(TEST_ENV_VALUE)
            .build();
    Protos.Environment newEnv = EnvUtils.withEnvVar(envBuilder.build(), TEST_ENV_KEY, TEST_REPLACE_ENV_VALUE);
    Assert.assertEquals(newEnv.getVariablesCount(), 1);
    Protos.Environment.Variable envVar = newEnv.getVariables(0);
    Assert.assertEquals(envVar.getValue(), TEST_REPLACE_ENV_VALUE);
}
Example 4
Source File: LaunchableTask.java    From incubator-heron with Apache License 2.0 5 votes vote down vote up 
protected Protos.Environment environment(Map<String, String> var) {
  Protos.Environment.Builder builder = Protos.Environment.newBuilder();

  for (Map.Entry<String, String> kv : var.entrySet()) {
    String key = kv.getKey();
    String value = kv.getValue();
    Protos.Environment.Variable variable =
        Protos.Environment.Variable.newBuilder().setName(key).setValue(value).build();
    builder.addVariables(variable);
  }

  return builder.build();
}
Example 5
Source File: TaskInfoFactory.java    From elasticsearch with Apache License 2.0 5 votes vote down vote up 
private Protos.CommandInfo dockerCommand(Configuration configuration, List<String> args, Long elasticSearchNodeId) {
    final Protos.Environment environment = Protos.Environment.newBuilder().addAllVariables(new ExecutorEnvironmentalVariables(configuration, elasticSearchNodeId).getList()).build();
    final Protos.CommandInfo.Builder builder = Protos.CommandInfo.newBuilder()
            .setShell(false)
            .mergeEnvironment(environment)
            .addAllArguments(args);
    if (!configuration.getElasticsearchSettingsLocation().isEmpty()) {
        builder.addUris(Protos.CommandInfo.URI.newBuilder().setValue(configuration.getElasticsearchSettingsLocation()));
    }
    return builder
            .build();
}
Example 6
Source File: ServiceCommandLineGenerator.java    From incubator-myriad with Apache License 2.0 5 votes vote down vote up 
protected Protos.Environment generateEnvironment(ServiceResourceProfile serviceResourceProfile, Collection<Long> ports) {
  Map<String, String> yarnEnv = myriadConfiguration.getYarnEnvironment();
  Protos.Environment.Builder builder = Protos.Environment.newBuilder();

  builder.addAllVariables(Iterables.transform(yarnEnv.entrySet(), new Function<Map.Entry<String, String>, Protos.Environment.Variable>() {
    public Protos.Environment.Variable apply(Map.Entry<String, String> x) {
      return Protos.Environment.Variable.newBuilder().setName(x.getKey()).setValue(x.getValue()).build();
    }
  }));

  StringBuilder hadoopOpts = new StringBuilder();
  String rmHostName = System.getProperty(KEY_YARN_RM_HOSTNAME);

  if (StringUtils.isNotEmpty(rmHostName)) {
    addJavaOpt(hadoopOpts, KEY_YARN_RM_HOSTNAME, rmHostName);
  }

  if (yarnEnv.containsKey(KEY_YARN_HOME)) {
    addJavaOpt(hadoopOpts, KEY_YARN_HOME, yarnEnv.get("YARN_HOME"));
  }

  Map<String, Long> portsMap = serviceResourceProfile.getPorts();
  Preconditions.checkState(portsMap.size() == ports.size());
  Iterator itr = ports.iterator();
  for (String portProperty : portsMap.keySet()) {
    addJavaOpt(hadoopOpts, portProperty, ALL_LOCAL_IPV4ADDR + itr.next());
  }

  if (myriadConfiguration.getYarnEnvironment().containsKey(ENV_HADOOP_OPTS)) {
    hadoopOpts.append(" ").append(yarnEnv.get(ENV_HADOOP_OPTS));
  }
  builder.addAllVariables(Collections.singleton(
          Protos.Environment.Variable.newBuilder()
              .setName(ENV_HADOOP_OPTS)
              .setValue(hadoopOpts.toString()).build())
  );
  return builder.build();
}
Example 7
Source File: TaskInfoFactory.java    From elasticsearch with Apache License 2.0 4 votes vote down vote up 
private Protos.ContainerInfo getContainer(Configuration configuration, Protos.TaskID taskID, Long elasticSearchNodeId, Protos.SlaveID slaveID) {
    final Protos.Environment environment = Protos.Environment.newBuilder().addAllVariables(new ExecutorEnvironmentalVariables(configuration, elasticSearchNodeId).getList()).build();
    final Protos.ContainerInfo.DockerInfo.Builder dockerInfo = Protos.ContainerInfo.DockerInfo.newBuilder()
            .addParameters(Protos.Parameter.newBuilder().setKey("env").setValue("MESOS_TASK_ID=" + taskID.getValue()))
            .setImage(configuration.getExecutorImage())
            .setForcePullImage(configuration.getExecutorForcePullImage())
            .setNetwork(Protos.ContainerInfo.DockerInfo.Network.HOST);
    // Add all env vars to container
    for (Protos.Environment.Variable variable : environment.getVariablesList()) {
        dockerInfo.addParameters(Protos.Parameter.newBuilder().setKey("env").setValue(variable.getName() + "=" + variable.getValue()));
    }

    final Protos.ContainerInfo.Builder builder = Protos.ContainerInfo.newBuilder()
            .setType(Protos.ContainerInfo.Type.DOCKER);

    if (configuration.getExternalVolumeDriver() != null && configuration.getExternalVolumeDriver().length() > 0) {

        LOGGER.debug("Is Docker Container and External Driver enabled");

        //docker external volume driver
        LOGGER.debug("Docker Driver: " + configuration.getExternalVolumeDriver());

        //note: this makes a unique data volume name per elastic search node
        StringBuffer sbData = new StringBuffer(configuration.getFrameworkName());
        sbData.append(Long.toString(elasticSearchNodeId));
        sbData.append("data:");
        sbData.append(Configuration.CONTAINER_PATH_DATA);
        String sHostPathOrExternalVolumeForData = sbData.toString();
        LOGGER.debug("Data Volume Name: " + sHostPathOrExternalVolumeForData);

        dockerInfo.addParameters(Protos.Parameter.newBuilder()
                .setKey("volume-driver")
                .setValue(configuration.getExternalVolumeDriver()));
        dockerInfo.addParameters(Protos.Parameter.newBuilder()
                .setKey("volume")
                .setValue(sHostPathOrExternalVolumeForData));
    } else {
        if (!configuration.getDataDir().isEmpty()) {
            builder.addVolumes(Protos.Volume.newBuilder()
                    .setHostPath(configuration.taskSpecificHostDir(slaveID))
                    .setContainerPath(Configuration.CONTAINER_PATH_DATA)
                    .setMode(Protos.Volume.Mode.RW)
                    .build());
        }
    }

    builder.setDocker(dockerInfo);

    if (!configuration.getElasticsearchSettingsLocation().isEmpty()) {
        final Path path = Paths.get(configuration.getElasticsearchSettingsLocation());
        final Path fileName = path.getFileName();
        if (fileName == null) {
            throw new IllegalArgumentException("Cannot parse filename from settings location. Please include the /elasticsearch.yml in the settings location.");
        }
        final String settingsFilename = fileName.toString();
        // Mount the custom yml file over the top of the standard elasticsearch.yml file.
        builder.addVolumes(Protos.Volume.newBuilder()
                .setHostPath("./" + settingsFilename) // Because the file has been uploaded by the uris.
                .setContainerPath(Configuration.CONTAINER_PATH_CONF_YML)
                .setMode(Protos.Volume.Mode.RO)
                .build());
    }

    return builder
            .build();
}
Example 8
Source File: NMExecutorCommandLineGenerator.java    From incubator-myriad with Apache License 2.0 4 votes vote down vote up 
protected Protos.Environment generateEnvironment(ServiceResourceProfile profile, Collection<Long> ports) {
  Map<String, String> yarnEnv = myriadConfiguration.getYarnEnvironment();
  Protos.Environment.Builder builder = Protos.Environment.newBuilder();
  builder.addAllVariables(Iterables.transform(yarnEnv.entrySet(), new Function<Map.Entry<String, String>, Protos.Environment.Variable>() {
    public Protos.Environment.Variable apply(Map.Entry<String, String> x) {
      return Protos.Environment.Variable.newBuilder().setName(x.getKey()).setValue(x.getValue()).build();
    }
  }));

  StringBuilder yarnOpts = new StringBuilder();
  String rmHostName = System.getProperty(KEY_YARN_RM_HOSTNAME);


  if (StringUtils.isNotEmpty(rmHostName)) {
    addJavaOpt(yarnOpts, KEY_YARN_RM_HOSTNAME, rmHostName);
  }

  if (yarnEnv.containsKey(KEY_YARN_HOME)) {
    addJavaOpt(yarnOpts, KEY_YARN_HOME, yarnEnv.get("YARN_HOME"));
  }

  addJavaOpt(yarnOpts, KEY_NM_RESOURCE_CPU_VCORES, Integer.toString(profile.getCpus().intValue()));
  addJavaOpt(yarnOpts, KEY_NM_RESOURCE_MEM_MB, Integer.toString(profile.getMemory().intValue()));
  Map<String, Long> portsMap = profile.getPorts();
  Preconditions.checkState(portsMap.size() == ports.size());

  Iterator itr = ports.iterator();
  for (String portProperty : portsMap.keySet()) {
    if (portProperty.endsWith("address")) {
      addJavaOpt(yarnOpts, portProperty, ALL_LOCAL_IPV4ADDR + itr.next());
    } else if (portProperty.endsWith("port")) {
      addJavaOpt(yarnOpts, portProperty, itr.next().toString());
    } else {
      LOGGER.warn("{} propery isn't an address or port!", portProperty);
    }
  }


  if (myriadConfiguration.getYarnEnvironment().containsKey(ENV_YARN_NODEMANAGER_OPTS)) {
    yarnOpts.append(" ").append(yarnEnv.get(ENV_YARN_NODEMANAGER_OPTS));
  }
  builder.addAllVariables(Collections.singleton(
          Protos.Environment.Variable.newBuilder()
              .setName(ENV_YARN_NODEMANAGER_OPTS)
              .setValue(yarnOpts.toString()).build())
  );
  return builder.build();
}