Java Code Examples for org.apache.hadoop.util.Shell#WINUTILS

The following examples show how to use org.apache.hadoop.util.Shell#WINUTILS . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: WindowsBasedProcessTree.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
public static boolean isAvailable() {
  if (Shell.WINDOWS) {
    ShellCommandExecutor shellExecutor = new ShellCommandExecutor(
        new String[] { Shell.WINUTILS, "help" });
    try {
      shellExecutor.execute();
    } catch (IOException e) {
      LOG.error(StringUtils.stringifyException(e));
    } finally {
      String output = shellExecutor.getOutput();
      if (output != null &&
          output.contains("Prints to stdout a list of processes in the task")) {
        return true;
      }
    }
  }
  return false;
}
Example 2
Source File: TestContainerExecutor.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Test (timeout = 5000)
public void testRunCommandWithCpuAndMemoryResources() {
  // Windows only test
  assumeTrue(Shell.WINDOWS);
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.NM_WINDOWS_CONTAINER_CPU_LIMIT_ENABLED, "true");
  conf.set(YarnConfiguration.NM_WINDOWS_CONTAINER_MEMORY_LIMIT_ENABLED, "true");
  String[] command = containerExecutor.getRunCommand("echo", "group1", null, null,
      conf, Resource.newInstance(1024, 1));
  float yarnProcessors = NodeManagerHardwareUtils.getContainersCores(
      ResourceCalculatorPlugin.getResourceCalculatorPlugin(null, conf),
      conf);
  int cpuRate = Math.min(10000, (int) ((1 * 10000) / yarnProcessors));
  // Assert the cpu and memory limits are set correctly in the command
  String[] expected = { Shell.WINUTILS, "task", "create", "-m", "1024", "-c",
      String.valueOf(cpuRate), "group1", "cmd /c " + "echo" };
  Assert.assertTrue(Arrays.equals(expected, command));
}
Example 3
Source File: TestContainerExecutor.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Test (timeout = 5000)
public void testRunCommandWithCpuAndMemoryResources() {
  // Windows only test
  assumeTrue(Shell.WINDOWS);
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.NM_WINDOWS_CONTAINER_CPU_LIMIT_ENABLED, "true");
  conf.set(YarnConfiguration.NM_WINDOWS_CONTAINER_MEMORY_LIMIT_ENABLED, "true");
  String[] command = containerExecutor.getRunCommand("echo", "group1", null, null,
      conf, Resource.newInstance(1024, 1));
  float yarnProcessors = NodeManagerHardwareUtils.getContainersCores(
      ResourceCalculatorPlugin.getResourceCalculatorPlugin(null, conf),
      conf);
  int cpuRate = Math.min(10000, (int) ((1 * 10000) / yarnProcessors));
  // Assert the cpu and memory limits are set correctly in the command
  String[] expected = { Shell.WINUTILS, "task", "create", "-m", "1024", "-c",
      String.valueOf(cpuRate), "group1", "cmd /c " + "echo" };
  Assert.assertTrue(Arrays.equals(expected, command));
}
Example 4
Source File: WindowsBasedProcessTree.java    From big-c with Apache License 2.0 6 votes vote down vote up 
public static boolean isAvailable() {
  if (Shell.WINDOWS) {
    ShellCommandExecutor shellExecutor = new ShellCommandExecutor(
        new String[] { Shell.WINUTILS, "help" });
    try {
      shellExecutor.execute();
    } catch (IOException e) {
      LOG.error(StringUtils.stringifyException(e));
    } finally {
      String output = shellExecutor.getOutput();
      if (output != null &&
          output.contains("Prints to stdout a list of processes in the task")) {
        return true;
      }
    }
  }
  return false;
}
Example 5
Source File: TestContainerLaunch.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Test (timeout = 10000)
public void testWindowsShellScriptBuilderLink() throws IOException {
  // Test is only relevant on Windows
  Assume.assumeTrue(Shell.WINDOWS);

  String linkCmd = "@" +Shell.WINUTILS + " symlink \"\" \"\"";

  // The tests are built on assuming 8191 max command line length
  assertEquals(8191, Shell.WINDOWS_MAX_SHELL_LENGHT);

  ShellScriptBuilder builder = ShellScriptBuilder.create();

  // test link
  builder.link(new Path(org.apache.commons.lang.StringUtils.repeat("A", 1024)),
      new Path(org.apache.commons.lang.StringUtils.repeat("B", 1024)));
  builder.link(
      new Path(org.apache.commons.lang.StringUtils.repeat(
          "E", (Shell.WINDOWS_MAX_SHELL_LENGHT - linkCmd.length())/2)),
      new Path(org.apache.commons.lang.StringUtils.repeat(
          "F", (Shell.WINDOWS_MAX_SHELL_LENGHT - linkCmd.length())/2)));
  try {
    builder.link(
        new Path(org.apache.commons.lang.StringUtils.repeat(
            "X", (Shell.WINDOWS_MAX_SHELL_LENGHT - linkCmd.length())/2 + 1)),
        new Path(org.apache.commons.lang.StringUtils.repeat(
            "Y", (Shell.WINDOWS_MAX_SHELL_LENGHT - linkCmd.length())/2) + 1));
    fail("long link was expected to throw");
  } catch(IOException e) {
    assertThat(e.getMessage(), containsString(expectedMessage));
  }
}
Example 6
Source File: TestContainerExecutor.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Test (timeout = 5000)
public void testRunCommandWithMemoryOnlyResources() {
  // Windows only test
  assumeTrue(Shell.WINDOWS);
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.NM_WINDOWS_CONTAINER_MEMORY_LIMIT_ENABLED, "true");
  String[] command = containerExecutor.getRunCommand("echo", "group1", null, null,
      conf, Resource.newInstance(1024, 1));
  // Assert the cpu and memory limits are set correctly in the command
  String[] expected = { Shell.WINUTILS, "task", "create", "-m", "1024", "-c",
      "-1", "group1", "cmd /c " + "echo" };
  Assert.assertTrue(Arrays.equals(expected, command));
}
Example 7
Source File: TestContainerExecutor.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Test (timeout = 5000)
public void testRunCommandWithNoResources() {
  // Windows only test
  assumeTrue(Shell.WINDOWS);
  Configuration conf = new Configuration();
  String[] command = containerExecutor.getRunCommand("echo", "group1", null, null,
      conf, Resource.newInstance(1024, 1));
  // Assert the cpu and memory limits are set correctly in the command
  String[] expected = { Shell.WINUTILS, "task", "create", "-m", "-1", "-c",
      "-1", "group1", "cmd /c " + "echo" };
  Assert.assertTrue(Arrays.equals(expected, command));
}
Example 8
Source File: WindowsSecureContainerExecutor.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Override
protected String[] getRunCommand(String command, String groupId,
    String userName, Path pidFile, Configuration conf) {
  File f = new File(command);
  if (LOG.isDebugEnabled()) {
    LOG.debug(String.format("getRunCommand: %s exists:%b", 
        command, f.exists()));
  }
  return new String[] { Shell.WINUTILS, "task", "createAsUser", groupId, 
      userName, pidFile.toString(), "cmd /c " + command };
}
Example 9
Source File: WindowsResourceCalculatorPlugin.java    From big-c with Apache License 2.0 5 votes vote down vote up 
String getSystemInfoInfoFromShell() {
  ShellCommandExecutor shellExecutor = new ShellCommandExecutor(
      new String[] { Shell.WINUTILS, "systeminfo" });
  try {
    shellExecutor.execute();
    return shellExecutor.getOutput();
  } catch (IOException e) {
    LOG.error(StringUtils.stringifyException(e));
  }
  return null;
}
Example 10
Source File: WindowsBasedProcessTree.java    From big-c with Apache License 2.0 5 votes vote down vote up 
String getAllProcessInfoFromShell() {
  ShellCommandExecutor shellExecutor = new ShellCommandExecutor(
      new String[] { Shell.WINUTILS, "task", "processList", taskProcessId });
  try {
    shellExecutor.execute();
    return shellExecutor.getOutput();
  } catch (IOException e) {
    LOG.error(StringUtils.stringifyException(e));
  }
  return null;
}
Example 11
Source File: TestContainerLaunch.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Test (timeout = 10000)
public void testWindowsShellScriptBuilderLink() throws IOException {
  // Test is only relevant on Windows
  Assume.assumeTrue(Shell.WINDOWS);

  String linkCmd = "@" +Shell.WINUTILS + " symlink \"\" \"\"";

  // The tests are built on assuming 8191 max command line length
  assertEquals(8191, Shell.WINDOWS_MAX_SHELL_LENGHT);

  ShellScriptBuilder builder = ShellScriptBuilder.create();

  // test link
  builder.link(new Path(org.apache.commons.lang.StringUtils.repeat("A", 1024)),
      new Path(org.apache.commons.lang.StringUtils.repeat("B", 1024)));
  builder.link(
      new Path(org.apache.commons.lang.StringUtils.repeat(
          "E", (Shell.WINDOWS_MAX_SHELL_LENGHT - linkCmd.length())/2)),
      new Path(org.apache.commons.lang.StringUtils.repeat(
          "F", (Shell.WINDOWS_MAX_SHELL_LENGHT - linkCmd.length())/2)));
  try {
    builder.link(
        new Path(org.apache.commons.lang.StringUtils.repeat(
            "X", (Shell.WINDOWS_MAX_SHELL_LENGHT - linkCmd.length())/2 + 1)),
        new Path(org.apache.commons.lang.StringUtils.repeat(
            "Y", (Shell.WINDOWS_MAX_SHELL_LENGHT - linkCmd.length())/2) + 1));
    fail("long link was expected to throw");
  } catch(IOException e) {
    assertThat(e.getMessage(), CoreMatchers.containsString(expectedMessage));
  }
}
Example 12
Source File: TestContainerExecutor.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Test (timeout = 5000)
public void testRunCommandWithMemoryOnlyResources() {
  // Windows only test
  assumeTrue(Shell.WINDOWS);
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.NM_WINDOWS_CONTAINER_MEMORY_LIMIT_ENABLED, "true");
  String[] command = containerExecutor.getRunCommand("echo", "group1", null, null,
      conf, Resource.newInstance(1024, 1));
  // Assert the cpu and memory limits are set correctly in the command
  String[] expected = { Shell.WINUTILS, "task", "create", "-m", "1024", "-c",
      "-1", "group1", "cmd /c " + "echo" };
  Assert.assertTrue(Arrays.equals(expected, command));
}
Example 13
Source File: TestContainerExecutor.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Test (timeout = 5000)
public void testRunCommandWithNoResources() {
  // Windows only test
  assumeTrue(Shell.WINDOWS);
  Configuration conf = new Configuration();
  String[] command = containerExecutor.getRunCommand("echo", "group1", null, null,
      conf, Resource.newInstance(1024, 1));
  // Assert the cpu and memory limits are set correctly in the command
  String[] expected = { Shell.WINUTILS, "task", "create", "-m", "-1", "-c",
      "-1", "group1", "cmd /c " + "echo" };
  Assert.assertTrue(Arrays.equals(expected, command));
}
Example 14
Source File: WindowsSecureContainerExecutor.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Override
protected String[] getRunCommand(String command, String groupId,
    String userName, Path pidFile, Configuration conf) {
  File f = new File(command);
  if (LOG.isDebugEnabled()) {
    LOG.debug(String.format("getRunCommand: %s exists:%b", 
        command, f.exists()));
  }
  return new String[] { Shell.WINUTILS, "task", "createAsUser", groupId, 
      userName, pidFile.toString(), "cmd /c " + command };
}
Example 15
Source File: WindowsResourceCalculatorPlugin.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
String getSystemInfoInfoFromShell() {
  ShellCommandExecutor shellExecutor = new ShellCommandExecutor(
      new String[] { Shell.WINUTILS, "systeminfo" });
  try {
    shellExecutor.execute();
    return shellExecutor.getOutput();
  } catch (IOException e) {
    LOG.error(StringUtils.stringifyException(e));
  }
  return null;
}
Example 16
Source File: WindowsBasedProcessTree.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
String getAllProcessInfoFromShell() {
  ShellCommandExecutor shellExecutor = new ShellCommandExecutor(
      new String[] { Shell.WINUTILS, "task", "processList", taskProcessId });
  try {
    shellExecutor.execute();
    return shellExecutor.getOutput();
  } catch (IOException e) {
    LOG.error(StringUtils.stringifyException(e));
  }
  return null;
}
Example 17
Source File: ContainerExecutor.java    From big-c with Apache License 2.0 4 votes vote down vote up 
/** 
 *  Return a command to execute the given command in OS shell.
 *  On Windows, the passed in groupId can be used to launch
 *  and associate the given groupId in a process group. On
 *  non-Windows, groupId is ignored. 
 */
protected String[] getRunCommand(String command, String groupId,
    String userName, Path pidFile, Configuration conf, Resource resource) {
  boolean containerSchedPriorityIsSet = false;
  int containerSchedPriorityAdjustment = 
      YarnConfiguration.DEFAULT_NM_CONTAINER_EXECUTOR_SCHED_PRIORITY;

  if (conf.get(YarnConfiguration.NM_CONTAINER_EXECUTOR_SCHED_PRIORITY) != 
      null) {
    containerSchedPriorityIsSet = true;
    containerSchedPriorityAdjustment = conf 
        .getInt(YarnConfiguration.NM_CONTAINER_EXECUTOR_SCHED_PRIORITY, 
        YarnConfiguration.DEFAULT_NM_CONTAINER_EXECUTOR_SCHED_PRIORITY);
  }

  if (Shell.WINDOWS) {
    int cpuRate = -1;
    int memory = -1;
    if (resource != null) {
      if (conf
          .getBoolean(
              YarnConfiguration.NM_WINDOWS_CONTAINER_MEMORY_LIMIT_ENABLED,
              YarnConfiguration.DEFAULT_NM_WINDOWS_CONTAINER_MEMORY_LIMIT_ENABLED)) {
        memory = resource.getMemory();
      }

      if (conf.getBoolean(
          YarnConfiguration.NM_WINDOWS_CONTAINER_CPU_LIMIT_ENABLED,
          YarnConfiguration.DEFAULT_NM_WINDOWS_CONTAINER_CPU_LIMIT_ENABLED)) {
        int containerVCores = resource.getVirtualCores();
        int nodeVCores = conf.getInt(YarnConfiguration.NM_VCORES,
            YarnConfiguration.DEFAULT_NM_VCORES);
        // cap overall usage to the number of cores allocated to YARN
        int nodeCpuPercentage = Math
            .min(
                conf.getInt(
                    YarnConfiguration.NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT,
                    YarnConfiguration.DEFAULT_NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT),
                100);
        nodeCpuPercentage = Math.max(0, nodeCpuPercentage);
        if (nodeCpuPercentage == 0) {
          String message = "Illegal value for "
              + YarnConfiguration.NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT
              + ". Value cannot be less than or equal to 0.";
          throw new IllegalArgumentException(message);
        }
        float yarnVCores = (nodeCpuPercentage * nodeVCores) / 100.0f;
        // CPU should be set to a percentage * 100, e.g. 20% cpu rate limit
        // should be set as 20 * 100. The following setting is equal to:
        // 100 * (100 * (vcores / Total # of cores allocated to YARN))
        cpuRate = Math.min(10000,
            (int) ((containerVCores * 10000) / yarnVCores));
      }
    }
    return new String[] { Shell.WINUTILS, "task", "create", "-m",
        String.valueOf(memory), "-c", String.valueOf(cpuRate), groupId,
        "cmd /c " + command };
  } else {
    List<String> retCommand = new ArrayList<String>();
    if (containerSchedPriorityIsSet) {
      retCommand.addAll(Arrays.asList("nice", "-n",
          Integer.toString(containerSchedPriorityAdjustment)));
    }
    retCommand.addAll(Arrays.asList("bash", command));
    return retCommand.toArray(new String[retCommand.size()]);
  }

}
Example 18
Source File: ContainerExecutor.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
/** 
 *  Return a command to execute the given command in OS shell.
 *  On Windows, the passed in groupId can be used to launch
 *  and associate the given groupId in a process group. On
 *  non-Windows, groupId is ignored. 
 */
protected String[] getRunCommand(String command, String groupId,
    String userName, Path pidFile, Configuration conf, Resource resource) {
  boolean containerSchedPriorityIsSet = false;
  int containerSchedPriorityAdjustment = 
      YarnConfiguration.DEFAULT_NM_CONTAINER_EXECUTOR_SCHED_PRIORITY;

  if (conf.get(YarnConfiguration.NM_CONTAINER_EXECUTOR_SCHED_PRIORITY) != 
      null) {
    containerSchedPriorityIsSet = true;
    containerSchedPriorityAdjustment = conf 
        .getInt(YarnConfiguration.NM_CONTAINER_EXECUTOR_SCHED_PRIORITY, 
        YarnConfiguration.DEFAULT_NM_CONTAINER_EXECUTOR_SCHED_PRIORITY);
  }

  if (Shell.WINDOWS) {
    int cpuRate = -1;
    int memory = -1;
    if (resource != null) {
      if (conf
          .getBoolean(
              YarnConfiguration.NM_WINDOWS_CONTAINER_MEMORY_LIMIT_ENABLED,
              YarnConfiguration.DEFAULT_NM_WINDOWS_CONTAINER_MEMORY_LIMIT_ENABLED)) {
        memory = resource.getMemory();
      }

      if (conf.getBoolean(
          YarnConfiguration.NM_WINDOWS_CONTAINER_CPU_LIMIT_ENABLED,
          YarnConfiguration.DEFAULT_NM_WINDOWS_CONTAINER_CPU_LIMIT_ENABLED)) {
        int containerVCores = resource.getVirtualCores();
        int nodeVCores = conf.getInt(YarnConfiguration.NM_VCORES,
            YarnConfiguration.DEFAULT_NM_VCORES);
        // cap overall usage to the number of cores allocated to YARN
        int nodeCpuPercentage = Math
            .min(
                conf.getInt(
                    YarnConfiguration.NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT,
                    YarnConfiguration.DEFAULT_NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT),
                100);
        nodeCpuPercentage = Math.max(0, nodeCpuPercentage);
        if (nodeCpuPercentage == 0) {
          String message = "Illegal value for "
              + YarnConfiguration.NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT
              + ". Value cannot be less than or equal to 0.";
          throw new IllegalArgumentException(message);
        }
        float yarnVCores = (nodeCpuPercentage * nodeVCores) / 100.0f;
        // CPU should be set to a percentage * 100, e.g. 20% cpu rate limit
        // should be set as 20 * 100. The following setting is equal to:
        // 100 * (100 * (vcores / Total # of cores allocated to YARN))
        cpuRate = Math.min(10000,
            (int) ((containerVCores * 10000) / yarnVCores));
      }
    }
    return new String[] { Shell.WINUTILS, "task", "create", "-m",
        String.valueOf(memory), "-c", String.valueOf(cpuRate), groupId,
        "cmd /c " + command };
  } else {
    List<String> retCommand = new ArrayList<String>();
    if (containerSchedPriorityIsSet) {
      retCommand.addAll(Arrays.asList("nice", "-n",
          Integer.toString(containerSchedPriorityAdjustment)));
    }
    retCommand.addAll(Arrays.asList("bash", command));
    return retCommand.toArray(new String[retCommand.size()]);
  }

}