Java Code Examples for org.apache.hadoop.mapreduce.TaskAttemptContext#progress()

The following examples show how to use org.apache.hadoop.mapreduce.TaskAttemptContext#progress() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: FileOutputCommitter.java    From RDFS with Apache License 2.0 6 votes vote down vote up 
/**
 * Move the files from the work directory to the job output directory
 * @param context the task context
 */
public void commitTask(TaskAttemptContext context) 
throws IOException {
  TaskAttemptID attemptId = context.getTaskAttemptID();
  if (workPath != null) {
    context.progress();
    if (outputFileSystem.exists(workPath)) {
      // Move the task outputs to their final place
      moveTaskOutputs(context, outputFileSystem, outputPath, workPath);
      // Delete the temporary task-specific output directory
      if (!outputFileSystem.delete(workPath, true)) {
        LOG.warn("Failed to delete the temporary output" + 
        " directory of task: " + attemptId + " - " + workPath);
      }
      LOG.info("Saved output of task '" + attemptId + "' to " + 
               outputPath);
    }
  }
}
Example 2
Source File: FileOutputCommitter.java    From hadoop-gpu with Apache License 2.0 6 votes vote down vote up 
/**
 * Move the files from the work directory to the job output directory
 * @param context the task context
 */
public void commitTask(TaskAttemptContext context) 
throws IOException {
  TaskAttemptID attemptId = context.getTaskAttemptID();
  if (workPath != null) {
    context.progress();
    if (outputFileSystem.exists(workPath)) {
      // Move the task outputs to their final place
      moveTaskOutputs(context, outputFileSystem, outputPath, workPath);
      // Delete the temporary task-specific output directory
      if (!outputFileSystem.delete(workPath, true)) {
        LOG.warn("Failed to delete the temporary output" + 
        " directory of task: " + attemptId + " - " + workPath);
      }
      LOG.info("Saved output of task '" + attemptId + "' to " + 
               outputPath);
    }
  }
}
Example 3
Source File: FileOutputCommitter.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Private
public void abortTask(TaskAttemptContext context, Path taskAttemptPath) throws IOException {
  if (hasOutputPath()) { 
    context.progress();
    if(taskAttemptPath == null) {
      taskAttemptPath = getTaskAttemptPath(context);
    }
    FileSystem fs = taskAttemptPath.getFileSystem(context.getConfiguration());
    if(!fs.delete(taskAttemptPath, true)) {
      LOG.warn("Could not delete "+taskAttemptPath);
    }
  } else {
    LOG.warn("Output Path is null in abortTask()");
  }
}
Example 4
Source File: FileOutputCommitter.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Private
public void abortTask(TaskAttemptContext context, Path taskAttemptPath) throws IOException {
  if (hasOutputPath()) { 
    context.progress();
    if(taskAttemptPath == null) {
      taskAttemptPath = getTaskAttemptPath(context);
    }
    FileSystem fs = taskAttemptPath.getFileSystem(context.getConfiguration());
    if(!fs.delete(taskAttemptPath, true)) {
      LOG.warn("Could not delete "+taskAttemptPath);
    }
  } else {
    LOG.warn("Output Path is null in abortTask()");
  }
}
Example 5
Source File: FileOutputCommitter.java    From RDFS with Apache License 2.0 5 votes vote down vote up 
/**
 * Delete the work directory
 */
@Override
public void abortTask(TaskAttemptContext context) throws IOException {
  try {
    if (workPath != null) { 
      context.progress();
      if (!outputFileSystem.delete(workPath, true)) {
        LOG.warn("Deleting output in " + workPath + " returns false");
      }
    }
  } catch (IOException ie) {
    LOG.warn("Error discarding output in " + workPath, ie);
    throw ie;
  }
}
Example 6
Source File: FileOutputCommitter.java    From hadoop-gpu with Apache License 2.0 5 votes vote down vote up 
/**
 * Delete the work directory
 */
@Override
public void abortTask(TaskAttemptContext context) {
  try {
    if (workPath != null) { 
      context.progress();
      outputFileSystem.delete(workPath, true);
    }
  } catch (IOException ie) {
    LOG.warn("Error discarding output" + StringUtils.stringifyException(ie));
  }
}
Example 7
Source File: FileOutputCommitter.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Private
public void commitTask(TaskAttemptContext context, Path taskAttemptPath) 
    throws IOException {

  TaskAttemptID attemptId = context.getTaskAttemptID();
  if (hasOutputPath()) {
    context.progress();
    if(taskAttemptPath == null) {
      taskAttemptPath = getTaskAttemptPath(context);
    }
    FileSystem fs = taskAttemptPath.getFileSystem(context.getConfiguration());
    FileStatus taskAttemptDirStatus;
    try {
      taskAttemptDirStatus = fs.getFileStatus(taskAttemptPath);
    } catch (FileNotFoundException e) {
      taskAttemptDirStatus = null;
    }

    if (taskAttemptDirStatus != null) {
      if (algorithmVersion == 1) {
        Path committedTaskPath = getCommittedTaskPath(context);
        if (fs.exists(committedTaskPath)) {
           if (!fs.delete(committedTaskPath, true)) {
             throw new IOException("Could not delete " + committedTaskPath);
           }
        }
        if (!fs.rename(taskAttemptPath, committedTaskPath)) {
          throw new IOException("Could not rename " + taskAttemptPath + " to "
              + committedTaskPath);
        }
        LOG.info("Saved output of task '" + attemptId + "' to " +
            committedTaskPath);
      } else {
        // directly merge everything from taskAttemptPath to output directory
        mergePaths(fs, taskAttemptDirStatus, outputPath);
        LOG.info("Saved output of task '" + attemptId + "' to " +
            outputPath);
      }
    } else {
      LOG.warn("No Output found for " + attemptId);
    }
  } else {
    LOG.warn("Output Path is null in commitTask()");
  }
}
Example 8
Source File: FileOutputCommitter.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Override
public void recoverTask(TaskAttemptContext context)
    throws IOException {
  if(hasOutputPath()) {
    context.progress();
    TaskAttemptID attemptId = context.getTaskAttemptID();
    int previousAttempt = getAppAttemptId(context) - 1;
    if (previousAttempt < 0) {
      throw new IOException ("Cannot recover task output for first attempt...");
    }

    Path previousCommittedTaskPath = getCommittedTaskPath(
        previousAttempt, context);
    FileSystem fs = previousCommittedTaskPath.getFileSystem(context.getConfiguration());
    if (LOG.isDebugEnabled()) {
      LOG.debug("Trying to recover task from " + previousCommittedTaskPath);
    }
    if (algorithmVersion == 1) {
      if (fs.exists(previousCommittedTaskPath)) {
        Path committedTaskPath = getCommittedTaskPath(context);
        if (fs.exists(committedTaskPath)) {
          if (!fs.delete(committedTaskPath, true)) {
            throw new IOException("Could not delete "+committedTaskPath);
          }
        }
        //Rename can fail if the parent directory does not yet exist.
        Path committedParent = committedTaskPath.getParent();
        fs.mkdirs(committedParent);
        if (!fs.rename(previousCommittedTaskPath, committedTaskPath)) {
          throw new IOException("Could not rename " + previousCommittedTaskPath +
              " to " + committedTaskPath);
        }
      } else {
          LOG.warn(attemptId+" had no output to recover.");
      }
    } else {
      // essentially a no-op, but for backwards compatibility
      // after upgrade to the new fileOutputCommitter,
      // check if there are any output left in committedTaskPath
      if (fs.exists(previousCommittedTaskPath)) {
        LOG.info("Recovering task for upgrading scenario, moving files from "
            + previousCommittedTaskPath + " to " + outputPath);
        FileStatus from = fs.getFileStatus(previousCommittedTaskPath);
        mergePaths(fs, from, outputPath);
      }
      LOG.info("Done recovering task " + attemptId);
    }
  } else {
    LOG.warn("Output Path is null in recoverTask()");
  }
}
Example 9
Source File: FileOutputCommitter.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Private
public void commitTask(TaskAttemptContext context, Path taskAttemptPath) 
    throws IOException {

  TaskAttemptID attemptId = context.getTaskAttemptID();
  if (hasOutputPath()) {
    context.progress();
    if(taskAttemptPath == null) {
      taskAttemptPath = getTaskAttemptPath(context);
    }
    FileSystem fs = taskAttemptPath.getFileSystem(context.getConfiguration());
    FileStatus taskAttemptDirStatus;
    try {
      taskAttemptDirStatus = fs.getFileStatus(taskAttemptPath);
    } catch (FileNotFoundException e) {
      taskAttemptDirStatus = null;
    }

    if (taskAttemptDirStatus != null) {
      if (algorithmVersion == 1) {
        Path committedTaskPath = getCommittedTaskPath(context);
        if (fs.exists(committedTaskPath)) {
           if (!fs.delete(committedTaskPath, true)) {
             throw new IOException("Could not delete " + committedTaskPath);
           }
        }
        if (!fs.rename(taskAttemptPath, committedTaskPath)) {
          throw new IOException("Could not rename " + taskAttemptPath + " to "
              + committedTaskPath);
        }
        LOG.info("Saved output of task '" + attemptId + "' to " +
            committedTaskPath);
      } else {
        // directly merge everything from taskAttemptPath to output directory
        mergePaths(fs, taskAttemptDirStatus, outputPath);
        LOG.info("Saved output of task '" + attemptId + "' to " +
            outputPath);
      }
    } else {
      LOG.warn("No Output found for " + attemptId);
    }
  } else {
    LOG.warn("Output Path is null in commitTask()");
  }
}
Example 10
Source File: FileOutputCommitter.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Override
public void recoverTask(TaskAttemptContext context)
    throws IOException {
  if(hasOutputPath()) {
    context.progress();
    TaskAttemptID attemptId = context.getTaskAttemptID();
    int previousAttempt = getAppAttemptId(context) - 1;
    if (previousAttempt < 0) {
      throw new IOException ("Cannot recover task output for first attempt...");
    }

    Path previousCommittedTaskPath = getCommittedTaskPath(
        previousAttempt, context);
    FileSystem fs = previousCommittedTaskPath.getFileSystem(context.getConfiguration());
    if (LOG.isDebugEnabled()) {
      LOG.debug("Trying to recover task from " + previousCommittedTaskPath);
    }
    if (algorithmVersion == 1) {
      if (fs.exists(previousCommittedTaskPath)) {
        Path committedTaskPath = getCommittedTaskPath(context);
        if (fs.exists(committedTaskPath)) {
          if (!fs.delete(committedTaskPath, true)) {
            throw new IOException("Could not delete "+committedTaskPath);
          }
        }
        //Rename can fail if the parent directory does not yet exist.
        Path committedParent = committedTaskPath.getParent();
        fs.mkdirs(committedParent);
        if (!fs.rename(previousCommittedTaskPath, committedTaskPath)) {
          throw new IOException("Could not rename " + previousCommittedTaskPath +
              " to " + committedTaskPath);
        }
      } else {
          LOG.warn(attemptId+" had no output to recover.");
      }
    } else {
      // essentially a no-op, but for backwards compatibility
      // after upgrade to the new fileOutputCommitter,
      // check if there are any output left in committedTaskPath
      if (fs.exists(previousCommittedTaskPath)) {
        LOG.info("Recovering task for upgrading scenario, moving files from "
            + previousCommittedTaskPath + " to " + outputPath);
        FileStatus from = fs.getFileStatus(previousCommittedTaskPath);
        mergePaths(fs, from, outputPath);
      }
      LOG.info("Done recovering task " + attemptId);
    }
  } else {
    LOG.warn("Output Path is null in recoverTask()");
  }
}