Java Code Examples for org.apache.hadoop.util.Shell.ShellCommandExecutor#execute()

public static boolean isAvailable() {
  if (Shell.WINDOWS) {
    ShellCommandExecutor shellExecutor = new ShellCommandExecutor(
        new String[] { Shell.WINUTILS, "help" });
    try {
      shellExecutor.execute();
    } catch (IOException e) {
      LOG.error(StringUtils.stringifyException(e));
    } finally {
      String output = shellExecutor.getOutput();
      if (output != null &&
          output.contains("Prints to stdout a list of processes in the task")) {
        return true;
      }
    }
  }
  return false;
}
 

/**
 * Change the permissions on a file / directory, recursively, if
 * needed.
 * @param filename name of the file whose permissions are to change
 * @param perm permission string
 * @param recursive true, if permissions should be changed recursively
 * @return the exit code from the command.
 * @throws IOException
 */
public static int chmod(String filename, String perm, boolean recursive)
                          throws IOException {
  String [] cmd = Shell.getSetPermissionCommand(perm, recursive);
  String[] args = new String[cmd.length + 1];
  System.arraycopy(cmd, 0, args, 0, cmd.length);
  args[cmd.length] = new File(filename).getPath();
  ShellCommandExecutor shExec = new ShellCommandExecutor(args);
  try {
    shExec.execute();
  }catch(IOException e) {
    if(LOG.isDebugEnabled()) {
      LOG.debug("Error while changing permission : " + filename 
                +" Exception: " + StringUtils.stringifyException(e));
    }
  }
  return shExec.getExitCode();
}
 

public void run() {
  try {
    Vector<String> args = new Vector<String>();
    if (isSetsidAvailable()) {
      args.add("setsid");
    }
    args.add("bash");
    args.add("-c");
    args.add(" echo $$ > " + pidFile + "; sh " + shellScript + " " + N
        + ";");
    shexec = new ShellCommandExecutor(args.toArray(new String[0]));
    shexec.execute();
  } catch (ExitCodeException ee) {
    LOG.info("Shell Command exit with a non-zero exit code. This is"
        + " expected as we are killing the subprocesses of the"
        + " task intentionally. " + ee);
  } catch (IOException ioe) {
    LOG.info("Error executing shell command " + ioe);
  } finally {
    LOG.info("Exit code: " + shexec.getExitCode());
  }
}
 

public void run() {
  try {
    Vector<String> args = new Vector<String>();
    if(ProcessTree.isSetsidAvailable) {
      args.add("setsid");
    }
    args.add("bash");
    args.add("-c");
    args.add(" echo $$ > " + pidFile + "; sh " +
                      shellScript + " " + N + ";") ;
    shexec = new ShellCommandExecutor(args.toArray(new String[0]));
    shexec.execute();
  } catch (ExitCodeException ee) {
    LOG.info("Shell Command exit with a non-zero exit code. This is" +
             " expected as we are killing the subprocesses of the" +
             " task intentionally. " + ee);
  } catch (IOException ioe) {
    LOG.info("Error executing shell command " + ioe);
  } finally {
    LOG.info("Exit code: " + shexec.getExitCode());
  }
}
 

/**
 * Convenience method used to sending appropriate Kill signal to the task 
 * VM
 * @param context
 * @param command
 * @throws IOException
 */
private void finishTask(TaskControllerContext context,
    TaskCommands command) throws IOException{
  if(context.task == null) {
    LOG.info("Context task null not killing the JVM");
    return;
  }
  ShellCommandExecutor shExec = buildTaskControllerExecutor(
      command, context.env.conf.getUser(), 
      buildKillTaskCommandArgs(context), context.env.workDir,
      context.env.env);
  try {
    shExec.execute();
  } catch (Exception e) {
    LOG.warn("Output from task-contoller is : " + shExec.getOutput());
    throw new IOException(e);
  }
}
 

/**
 * Change the permissions on a file / directory, recursively, if
 * needed.
 * @param filename name of the file whose permissions are to change
 * @param perm permission string
 * @param recursive true, if permissions should be changed recursively
 * @return the exit code from the command.
 * @throws IOException
 */
public static int chmod(String filename, String perm, boolean recursive)
                          throws IOException {
  String [] cmd = Shell.getSetPermissionCommand(perm, recursive);
  String[] args = new String[cmd.length + 1];
  System.arraycopy(cmd, 0, args, 0, cmd.length);
  args[cmd.length] = new File(filename).getPath();
  ShellCommandExecutor shExec = new ShellCommandExecutor(args);
  try {
    shExec.execute();
  }catch(IOException e) {
    if(LOG.isDebugEnabled()) {
      LOG.debug("Error while changing permission : " + filename 
                +" Exception: " + StringUtils.stringifyException(e));
    }
  }
  return shExec.getExitCode();
}
 

/**
 * Is perl supported on this machine ?
 * @return true if perl is available and is working as expected
 */
public static boolean hasPerlSupport() {
  boolean hasPerl = false;
  ShellCommandExecutor shexec = new ShellCommandExecutor(
    new String[] { "perl", "-e", "print 42" });
  try {
    shexec.execute();
    if (shexec.getOutput().equals("42")) {
      hasPerl = true;
    }
    else {
      LOG.warn("Perl is installed, but isn't behaving as expected.");
    }
  } catch (Exception e) {
    LOG.warn("Could not run perl: " + e);
  }
  return hasPerl;
}
 

public void mountCgroups(List<String> cgroupKVs, String hierarchy)
       throws IOException {
  List<String> command = new ArrayList<String>(
          Arrays.asList(containerExecutorExe, "--mount-cgroups", hierarchy));
  command.addAll(cgroupKVs);
  
  String[] commandArray = command.toArray(new String[command.size()]);
  ShellCommandExecutor shExec = new ShellCommandExecutor(commandArray);

  if (LOG.isDebugEnabled()) {
      LOG.debug("mountCgroups: " + Arrays.toString(commandArray));
  }

  try {
      shExec.execute();
  } catch (IOException e) {
      int ret_code = shExec.getExitCode();
      LOG.warn("Exception in LinuxContainerExecutor mountCgroups ", e);
      logOutput(shExec.getOutput());
      throw new IOException("Problem mounting cgroups " + cgroupKVs + 
        "; exit code = " + ret_code + " and output: " + shExec.getOutput(), e);
  }
}
 

@Override 
public void init() throws IOException {        
  // Send command to executor which will just start up, 
  // verify configuration/permissions and exit
  List<String> command = new ArrayList<String>(
      Arrays.asList(containerExecutorExe,
          "--checksetup"));
  String[] commandArray = command.toArray(new String[command.size()]);
  ShellCommandExecutor shExec = new ShellCommandExecutor(commandArray);
  if (LOG.isDebugEnabled()) {
    LOG.debug("checkLinuxExecutorSetup: " + Arrays.toString(commandArray));
  }
  try {
    shExec.execute();
  } catch (ExitCodeException e) {
    int exitCode = shExec.getExitCode();
    LOG.warn("Exit code from container executor initialization is : "
        + exitCode, e);
    logOutput(shExec.getOutput());
    throw new IOException("Linux container executor not configured properly"
        + " (error=" + exitCode + ")", e);
  }
 
  resourcesHandler.init(this);
}
 

String getAllProcessInfoFromShell() {
  ShellCommandExecutor shellExecutor = new ShellCommandExecutor(
      new String[] { Shell.WINUTILS, "task", "processList", taskProcessId });
  try {
    shellExecutor.execute();
    return shellExecutor.getOutput();
  } catch (IOException e) {
    LOG.error(StringUtils.stringifyException(e));
  }
  return null;
}
 

private static void unTarUsingTar(File inFile, File untarDir,
    boolean gzipped) throws IOException {
  StringBuffer untarCommand = new StringBuffer();
  if (gzipped) {
    untarCommand.append(" gzip -dc '");
    untarCommand.append(FileUtil.makeShellPath(inFile));
    untarCommand.append("' | (");
  } 
  untarCommand.append("cd '");
  untarCommand.append(FileUtil.makeShellPath(untarDir)); 
  untarCommand.append("' ; ");
  untarCommand.append("tar -xf ");

  if (gzipped) {
    untarCommand.append(" -)");
  } else {
    untarCommand.append(FileUtil.makeShellPath(inFile));
  }
  String[] shellCmd = { "bash", "-c", untarCommand.toString() };
  ShellCommandExecutor shexec = new ShellCommandExecutor(shellCmd);
  shexec.execute();
  int exitcode = shexec.getExitCode();
  if (exitcode != 0) {
    throw new IOException("Error untarring file " + inFile + 
                ". Tar process exited with exit code " + exitcode);
  }
}
 

/**
 * Runs the script given in args
 * @param args script name followed by its argumnets
 * @param dir current working directory.
 * @throws IOException
 */
public void runScript(List<String> args, File dir) throws IOException {
  ShellCommandExecutor shexec = 
          new ShellCommandExecutor(args.toArray(new String[0]), dir);
  shexec.execute();
  int exitCode = shexec.getExitCode();
  if (exitCode != 0) {
    throw new IOException("Task debug script exit with nonzero status of " 
                          + exitCode + ".");
  }
}
 

String getAllProcessInfoFromShell() {
  ShellCommandExecutor shellExecutor = new ShellCommandExecutor(
      new String[] { Shell.WINUTILS, "task", "processList", taskProcessId });
  try {
    shellExecutor.execute();
    return shellExecutor.getOutput();
  } catch (IOException e) {
    LOG.error(StringUtils.stringifyException(e));
  }
  return null;
}
 

/**
 * Execute "ps -eo pid,ppid,pcpu,rss,command"
 * @return String[] which contains the execution result
 */
protected String[] getPS() {
  ShellCommandExecutor shellExecutor = new ShellCommandExecutor(CMD);
  try {
    shellExecutor.execute();
  }
  catch (IOException e) {
    LOG.error(StringUtils.stringifyException(e));
    return null;
  }
  return shellExecutor.getOutput().split("\n");
}
 

private String runResolveCommand(List<String> args) {
  int loopCount = 0;
  if (args.size() == 0) {
    return null;
  }
  StringBuffer allOutput = new StringBuffer();
  int numProcessed = 0;
  if (maxArgs < MIN_ALLOWABLE_ARGS) {
    LOG.warn("Invalid value " + Integer.toString(maxArgs)
        + " for " + SCRIPT_ARG_COUNT_KEY + "; must be >= "
        + Integer.toString(MIN_ALLOWABLE_ARGS));
    return null;
  }
  
  while (numProcessed != args.size()) {
    int start = maxArgs * loopCount;
    List <String> cmdList = new ArrayList<String>();
    cmdList.add(scriptName);
    for (numProcessed = start; numProcessed < (start + maxArgs) && 
         numProcessed < args.size(); numProcessed++) {
      cmdList.add(args.get(numProcessed)); 
    }
    File dir = null;
    String userDir;
    if ((userDir = System.getProperty("user.dir")) != null) {
      dir = new File(userDir);
    }
    ShellCommandExecutor s = new ShellCommandExecutor(
                                 cmdList.toArray(new String[0]), dir);
    try {
      s.execute();
      allOutput.append(s.getOutput() + " ");
    } catch (Exception e) {
      LOG.warn(StringUtils.stringifyException(e));
      return null;
    }
    loopCount++; 
  }
  return allOutput.toString();
}
 

private static void unTarUsingTar(File inFile, File untarDir,
                                  boolean gzipped) throws IOException {
  StringBuffer untarCommand = new StringBuffer();
  if (gzipped) {
    untarCommand.append(" gzip -dc '");
    untarCommand.append(FileUtil.makeSecureShellPath(inFile));
    untarCommand.append("' | (");
  }
  untarCommand.append("cd '");
  untarCommand.append(FileUtil.makeSecureShellPath(untarDir));
  untarCommand.append("' && ");
  untarCommand.append("tar -xf ");

  if (gzipped) {
    untarCommand.append(" -)");
  } else {
    untarCommand.append(FileUtil.makeSecureShellPath(inFile));
  }
  String[] shellCmd = { "bash", "-c", untarCommand.toString() };
  ShellCommandExecutor shexec = new ShellCommandExecutor(shellCmd);
  shexec.execute();
  int exitcode = shexec.getExitCode();
  if (exitcode != 0) {
    throw new IOException("Error untarring file " + inFile +
        ". Tar process exited with exit code " + exitcode);
  }
}
 

/**
 * Launch a task JVM that will run as the owner of the job.
 * 
 * This method launches a task JVM by executing a setuid
 * executable that will switch to the user and run the
 * task.
 */
@Override
void launchTaskJVM(TaskController.TaskControllerContext context) 
                                      throws IOException {
  JvmEnv env = context.env;
  // get the JVM command line.
  String cmdLine = 
    TaskLog.buildCommandLine(env.setup, env.vargs, env.stdout, env.stderr,
        env.logSize, true);

  StringBuffer sb = new StringBuffer();
  //export out all the environment variable before child command as
  //the setuid/setgid binaries would not be getting, any environmental
  //variables which begin with LD_*.
  for(Entry<String, String> entry : env.env.entrySet()) {
    sb.append("export ");
    sb.append(entry.getKey());
    sb.append("=");
    sb.append(entry.getValue());
    sb.append("\n");
  }
  sb.append(cmdLine);
  // write the command to a file in the
  // task specific cache directory
  writeCommand(sb.toString(), getTaskCacheDirectory(context));
  
  // Call the taskcontroller with the right parameters.
  List<String> launchTaskJVMArgs = buildLaunchTaskArgs(context);
  ShellCommandExecutor shExec =  buildTaskControllerExecutor(
                                  TaskCommands.LAUNCH_TASK_JVM, 
                                  env.conf.getUser(),
                                  launchTaskJVMArgs, env.workDir, env.env);
  context.shExec = shExec;
  try {
    shExec.execute();
  } catch (Exception e) {
    LOG.warn("Exception thrown while launching task JVM : " + 
        StringUtils.stringifyException(e));
    LOG.warn("Exit code from task is : " + shExec.getExitCode());
    LOG.warn("Output from task-contoller is : " + shExec.getOutput());
    throw new IOException(e);
  }
  if(LOG.isDebugEnabled()) {
    LOG.debug("output after executing task jvm = " + shExec.getOutput()); 
  }
}
 

@Override
public void startLocalizer(Path nmPrivateContainerTokensPath,
    InetSocketAddress nmAddr, String user, String appId, String locId,
    LocalDirsHandlerService dirsHandler)
    throws IOException, InterruptedException {

  List<String> localDirs = dirsHandler.getLocalDirs();
  List<String> logDirs = dirsHandler.getLogDirs();
  
  verifyUsernamePattern(user);
  String runAsUser = getRunAsUser(user);
  List<String> command = new ArrayList<String>();
  addSchedPriorityCommand(command);
  command.addAll(Arrays.asList(containerExecutorExe, 
                 runAsUser,
                 user, 
                 Integer.toString(Commands.INITIALIZE_CONTAINER.getValue()),
                 appId,
                 nmPrivateContainerTokensPath.toUri().getPath().toString(),
                 StringUtils.join(",", localDirs),
                 StringUtils.join(",", logDirs)));

  File jvm =                                  // use same jvm as parent
    new File(new File(System.getProperty("java.home"), "bin"), "java");
  command.add(jvm.toString());
  command.add("-classpath");
  command.add(System.getProperty("java.class.path"));
  String javaLibPath = System.getProperty("java.library.path");
  if (javaLibPath != null) {
    command.add("-Djava.library.path=" + javaLibPath);
  }
  buildMainArgs(command, user, appId, locId, nmAddr, localDirs);
  String[] commandArray = command.toArray(new String[command.size()]);
  ShellCommandExecutor shExec = new ShellCommandExecutor(commandArray);
  if (LOG.isDebugEnabled()) {
    LOG.debug("initApplication: " + Arrays.toString(commandArray));
  }
  try {
    shExec.execute();
    if (LOG.isDebugEnabled()) {
      logOutput(shExec.getOutput());
    }
  } catch (ExitCodeException e) {
    int exitCode = shExec.getExitCode();
    LOG.warn("Exit code from container " + locId + " startLocalizer is : "
        + exitCode, e);
    logOutput(shExec.getOutput());
    throw new IOException("Application " + appId + " initialization failed" +
    		" (exitCode=" + exitCode + ") with output: " + shExec.getOutput(), e);
  }
}
 

/**
 * Build and execute the resolution command. The command is
 * executed in the directory specified by the system property
 * "user.dir" if set; otherwise the current working directory is used
 * @param args a list of arguments
 * @return null if the number of arguments is out of range,
 * or the output of the command.
 */
protected String runResolveCommand(List<String> args, 
    String commandScriptName) {
  int loopCount = 0;
  if (args.size() == 0) {
    return null;
  }
  StringBuilder allOutput = new StringBuilder();
  int numProcessed = 0;
  if (maxArgs < MIN_ALLOWABLE_ARGS) {
    LOG.warn("Invalid value " + Integer.toString(maxArgs)
        + " for " + SCRIPT_ARG_COUNT_KEY + "; must be >= "
        + Integer.toString(MIN_ALLOWABLE_ARGS));
    return null;
  }

  while (numProcessed != args.size()) {
    int start = maxArgs * loopCount;
    List<String> cmdList = new ArrayList<String>();
    cmdList.add(commandScriptName);
    for (numProcessed = start; numProcessed < (start + maxArgs) &&
        numProcessed < args.size(); numProcessed++) {
      cmdList.add(args.get(numProcessed));
    }
    File dir = null;
    String userDir;
    if ((userDir = System.getProperty("user.dir")) != null) {
      dir = new File(userDir);
    }
    ShellCommandExecutor s = new ShellCommandExecutor(
        cmdList.toArray(new String[cmdList.size()]), dir);
    try {
      s.execute();
      allOutput.append(s.getOutput()).append(" ");
    } catch (Exception e) {
      LOG.warn("Exception running " + s, e);
      return null;
    }
    loopCount++;
  }
  return allOutput.toString();
}
 

@Override
public void deleteAsUser(String user, Path dir, Path... baseDirs) {
  verifyUsernamePattern(user);
  String runAsUser = getRunAsUser(user);

  String dirString = dir == null ? "" : dir.toUri().getPath();

  List<String> command = new ArrayList<String>(
      Arrays.asList(containerExecutorExe,
                  runAsUser,
                  user,
                  Integer.toString(Commands.DELETE_AS_USER.getValue()),
                  dirString));
  List<String> pathsToDelete = new ArrayList<String>();
  if (baseDirs == null || baseDirs.length == 0) {
    LOG.info("Deleting absolute path : " + dir);
    pathsToDelete.add(dirString);
  } else {
    for (Path baseDir : baseDirs) {
      Path del = dir == null ? baseDir : new Path(baseDir, dir);
      LOG.info("Deleting path : " + del);
      pathsToDelete.add(del.toString());
      command.add(baseDir.toUri().getPath());
    }
  }
  String[] commandArray = command.toArray(new String[command.size()]);
  ShellCommandExecutor shExec = new ShellCommandExecutor(commandArray);
  if (LOG.isDebugEnabled()) {
    LOG.debug("deleteAsUser: " + Arrays.toString(commandArray));
  }
  try {
    shExec.execute();
    if (LOG.isDebugEnabled()) {
      logOutput(shExec.getOutput());
    }
  } catch (IOException e) {
    int exitCode = shExec.getExitCode();
    LOG.error("DeleteAsUser for " + StringUtils.join(" ", pathsToDelete)
        + " returned with exit code: " + exitCode, e);
    LOG.error("Output from LinuxContainerExecutor's deleteAsUser follows:");
    logOutput(shExec.getOutput());
  }
}