Java Code Examples for org.apache.hadoop.mapred.JobClient#getConf()

The following examples show how to use org.apache.hadoop.mapred.JobClient#getConf() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: WriteIopsCalculator.java    From emr-dynamodb-connector with Apache License 2.0 6 votes vote down vote up 
public WriteIopsCalculator(JobClient jobClient, DynamoDBClient dynamoDBClient, String tableName) {
  this.jobConf = (JobConf) jobClient.getConf();
  this.jobClient = jobClient;

  this.dynamoDBClient = dynamoDBClient;
  this.tableName = tableName;
  this.throughputPercent = Double.parseDouble(jobConf
      .get(DynamoDBConstants.THROUGHPUT_WRITE_PERCENT,
          DynamoDBConstants.DEFAULT_THROUGHPUT_PERCENTAGE));

  log.info("Table name: " + tableName);
  log.info("Throughput percent: " + throughputPercent);

  String taskId = jobConf.get("mapreduce.task.attempt.id");
  log.info("Task Id: " + taskId);
  if (Strings.isNullOrEmpty(taskId)) {
    // Running in local mode
    maxParallelTasks = 1;
  } else {
    int totalMapTasks = jobConf.getNumMapTasks();
    log.info("Total map tasks: " + totalMapTasks);

    maxParallelTasks = Math.min(calculateMaxMapTasks(totalMapTasks), totalMapTasks);
    log.info("Max parallel map tasks: " + maxParallelTasks);
  }
}
Example 2
Source File: ReadIopsCalculator.java    From emr-dynamodb-connector with Apache License 2.0 6 votes vote down vote up 
public ReadIopsCalculator(JobClient jobClient, DynamoDBClient dynamoDBClient, String tableName,
    int totalSegments, int localSegments) {
  this.jobConf = (JobConf) jobClient.getConf();
  this.jobClient = jobClient;

  this.dynamoDBClient = dynamoDBClient;
  this.tableName = tableName;
  this.totalSegments = totalSegments;
  this.localSegments = localSegments;

  this.throughputPercent = Double.parseDouble(jobConf.get(DynamoDBConstants
      .THROUGHPUT_READ_PERCENT, DynamoDBConstants.DEFAULT_THROUGHPUT_PERCENTAGE));

  log.info("Table name: " + tableName);
  log.info("Throughput percent: " + throughputPercent);
}
Example 3
Source File: DynamoDBUtil.java    From emr-dynamodb-connector with Apache License 2.0 5 votes vote down vote up 
public static int calcMaxMapTasks(JobClient jobClient) throws IOException {
  JobConf conf = (JobConf) jobClient.getConf();
  NodeCapacityProvider nodeCapacityProvider = new ClusterTopologyNodeCapacityProvider(conf);
  YarnContainerAllocator yarnContainerAllocator = new RoundRobinYarnContainerAllocator();
  TaskCalculator taskCalculator = new TaskCalculator(jobClient, nodeCapacityProvider,
      yarnContainerAllocator);
  return taskCalculator.getMaxMapTasks();
}