org.apache.hadoop.conf.Configuration.IntegerRanges Java Examples

The following examples show how to use org.apache.hadoop.conf.Configuration.IntegerRanges. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: Job.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
private void printTaskEvents(TaskCompletionEvent[] events,
    Job.TaskStatusFilter filter, boolean profiling, IntegerRanges mapRanges,
    IntegerRanges reduceRanges) throws IOException, InterruptedException {
  for (TaskCompletionEvent event : events) {
    switch (filter) {
    case NONE:
      break;
    case SUCCEEDED:
      if (event.getStatus() == 
        TaskCompletionEvent.Status.SUCCEEDED) {
        LOG.info(event.toString());
      }
      break; 
    case FAILED:
      if (event.getStatus() == 
        TaskCompletionEvent.Status.FAILED) {
        LOG.info(event.toString());
        // Displaying the task diagnostic information
        TaskAttemptID taskId = event.getTaskAttemptId();
        String[] taskDiagnostics = getTaskDiagnostics(taskId); 
        if (taskDiagnostics != null) {
          for (String diagnostics : taskDiagnostics) {
            System.err.println(diagnostics);
          }
        }
      }
      break; 
    case KILLED:
      if (event.getStatus() == TaskCompletionEvent.Status.KILLED){
        LOG.info(event.toString());
      }
      break; 
    case ALL:
      LOG.info(event.toString());
      break;
    }
  }
}
Example #2
Source File: TestConfiguration.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public void testIntegerRanges() {
  Configuration conf = new Configuration();
  conf.set("first", "-100");
  conf.set("second", "4-6,9-10,27");
  conf.set("third", "34-");
  Configuration.IntegerRanges range = conf.getRange("first", null);
  System.out.println("first = " + range);
  assertEquals(true, range.isIncluded(0));
  assertEquals(true, range.isIncluded(1));
  assertEquals(true, range.isIncluded(100));
  assertEquals(false, range.isIncluded(101));
  range = conf.getRange("second", null);
  System.out.println("second = " + range);
  assertEquals(false, range.isIncluded(3));
  assertEquals(true, range.isIncluded(4));
  assertEquals(true, range.isIncluded(6));
  assertEquals(false, range.isIncluded(7));
  assertEquals(false, range.isIncluded(8));
  assertEquals(true, range.isIncluded(9));
  assertEquals(true, range.isIncluded(10));
  assertEquals(false, range.isIncluded(11));
  assertEquals(false, range.isIncluded(26));
  assertEquals(true, range.isIncluded(27));
  assertEquals(false, range.isIncluded(28));
  range = conf.getRange("third", null);
  System.out.println("third = " + range);
  assertEquals(false, range.isIncluded(33));
  assertEquals(true, range.isIncluded(34));
  assertEquals(true, range.isIncluded(100000000));
}
Example #3
Source File: TestConfiguration.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public void testIntegerRanges() {
  Configuration conf = new Configuration();
  conf.set("first", "-100");
  conf.set("second", "4-6,9-10,27");
  conf.set("third", "34-");
  Configuration.IntegerRanges range = conf.getRange("first", null);
  System.out.println("first = " + range);
  assertEquals(true, range.isIncluded(0));
  assertEquals(true, range.isIncluded(1));
  assertEquals(true, range.isIncluded(100));
  assertEquals(false, range.isIncluded(101));
  range = conf.getRange("second", null);
  System.out.println("second = " + range);
  assertEquals(false, range.isIncluded(3));
  assertEquals(true, range.isIncluded(4));
  assertEquals(true, range.isIncluded(6));
  assertEquals(false, range.isIncluded(7));
  assertEquals(false, range.isIncluded(8));
  assertEquals(true, range.isIncluded(9));
  assertEquals(true, range.isIncluded(10));
  assertEquals(false, range.isIncluded(11));
  assertEquals(false, range.isIncluded(26));
  assertEquals(true, range.isIncluded(27));
  assertEquals(false, range.isIncluded(28));
  range = conf.getRange("third", null);
  System.out.println("third = " + range);
  assertEquals(false, range.isIncluded(33));
  assertEquals(true, range.isIncluded(34));
  assertEquals(true, range.isIncluded(100000000));
}
Example #4
Source File: Job.java    From big-c with Apache License 2.0 5 votes vote down vote up 
private void printTaskEvents(TaskCompletionEvent[] events,
    Job.TaskStatusFilter filter, boolean profiling, IntegerRanges mapRanges,
    IntegerRanges reduceRanges) throws IOException, InterruptedException {
  for (TaskCompletionEvent event : events) {
    switch (filter) {
    case NONE:
      break;
    case SUCCEEDED:
      if (event.getStatus() == 
        TaskCompletionEvent.Status.SUCCEEDED) {
        LOG.info(event.toString());
      }
      break; 
    case FAILED:
      if (event.getStatus() == 
        TaskCompletionEvent.Status.FAILED) {
        LOG.info(event.toString());
        // Displaying the task diagnostic information
        TaskAttemptID taskId = event.getTaskAttemptId();
        String[] taskDiagnostics = getTaskDiagnostics(taskId); 
        if (taskDiagnostics != null) {
          for (String diagnostics : taskDiagnostics) {
            System.err.println(diagnostics);
          }
        }
      }
      break; 
    case KILLED:
      if (event.getStatus() == TaskCompletionEvent.Status.KILLED){
        LOG.info(event.toString());
      }
      break; 
    case ALL:
      LOG.info(event.toString());
      break;
    }
  }
}
Example #5
Source File: MockupMapContext.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
public IntegerRanges getProfileTaskRange(boolean isMap) {
    throw new NotImplementedException();
}
Example #6
Source File: SMHIveContextWrapper.java    From spliceengine with GNU Affero General Public License v3.0 4 votes vote down vote up 
public IntegerRanges getProfileTaskRange(boolean isMap) {
	// TODO Auto-generated method stub
	return null;
}
Example #7
Source File: NativeMapContext.java    From geowave with Apache License 2.0 4 votes vote down vote up 
@Override
public IntegerRanges getProfileTaskRange(final boolean isMap) {
  return context.getProfileTaskRange(isMap);
}
Example #8
Source File: NativeReduceContext.java    From geowave with Apache License 2.0 4 votes vote down vote up 
@Override
public IntegerRanges getProfileTaskRange(final boolean isMap) {
  return writableContext.getProfileTaskRange(isMap);
}
Example #9
Source File: HttpServer2.java    From knox with Apache License 2.0 4 votes vote down vote up 
public Builder setPortRanges(IntegerRanges ranges) {
  this.portRanges = ranges;
  return this;
}
Example #10
Source File: HttpServer2.java    From knox with Apache License 2.0 4 votes vote down vote up 
public Builder setPortRanges(IntegerRanges ranges) {
  this.portRanges = ranges;
  return this;
}
Example #11
Source File: MockupMapContext.java    From kylin with Apache License 2.0 4 votes vote down vote up 
public IntegerRanges getProfileTaskRange(boolean isMap) {
    throw new NotImplementedException();
}
Example #12
Source File: MockupMapContext.java    From kylin with Apache License 2.0 4 votes vote down vote up 
public IntegerRanges getProfileTaskRange(boolean isMap) {
    throw new NotImplementedException();
}
Example #13
Source File: HttpServer2.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up 
public Builder setPortRanges(IntegerRanges ranges) {
  this.portRanges = ranges;
  return this;
}
Example #14
Source File: ChainMapContextImpl.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Override
public IntegerRanges getProfileTaskRange(boolean isMap) {
  return base.getProfileTaskRange(isMap);
}
Example #15
Source File: ChainReduceContextImpl.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Override
public IntegerRanges getProfileTaskRange(boolean isMap) {
  return base.getProfileTaskRange(isMap);
}
Example #16
Source File: WrappedReducer.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Override
public IntegerRanges getProfileTaskRange(boolean isMap) {
  return reduceContext.getProfileTaskRange(isMap);
}
Example #17
Source File: WrappedMapper.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Override
public IntegerRanges getProfileTaskRange(boolean isMap) {
  return mapContext.getProfileTaskRange(isMap);
}
Example #18
Source File: Job.java    From big-c with Apache License 2.0 4 votes vote down vote up 
/**
 * Monitor a job and print status in real-time as progress is made and tasks 
 * fail.
 * @return true if the job succeeded
 * @throws IOException if communication to the JobTracker fails
 */
public boolean monitorAndPrintJob() 
    throws IOException, InterruptedException {
  String lastReport = null;
  Job.TaskStatusFilter filter;
  Configuration clientConf = getConfiguration();
  filter = Job.getTaskOutputFilter(clientConf);
  JobID jobId = getJobID();
  LOG.info("Running job: " + jobId);
  int eventCounter = 0;
  boolean profiling = getProfileEnabled();
  IntegerRanges mapRanges = getProfileTaskRange(true);
  IntegerRanges reduceRanges = getProfileTaskRange(false);
  int progMonitorPollIntervalMillis = 
    Job.getProgressPollInterval(clientConf);
  /* make sure to report full progress after the job is done */
  boolean reportedAfterCompletion = false;
  boolean reportedUberMode = false;
  while (!isComplete() || !reportedAfterCompletion) {
    if (isComplete()) {
      reportedAfterCompletion = true;
    } else {
      Thread.sleep(progMonitorPollIntervalMillis);
    }
    if (status.getState() == JobStatus.State.PREP) {
      continue;
    }      
    if (!reportedUberMode) {
      reportedUberMode = true;
      LOG.info("Job " + jobId + " running in uber mode : " + isUber());
    }      
    String report = 
      (" map " + StringUtils.formatPercent(mapProgress(), 0)+
          " reduce " + 
          StringUtils.formatPercent(reduceProgress(), 0));
    if (!report.equals(lastReport)) {
      LOG.info(report);
      lastReport = report;
    }

    TaskCompletionEvent[] events = 
      getTaskCompletionEvents(eventCounter, 10); 
    eventCounter += events.length;
    printTaskEvents(events, filter, profiling, mapRanges, reduceRanges);
  }
  boolean success = isSuccessful();
  if (success) {
    LOG.info("Job " + jobId + " completed successfully");
  } else {
    LOG.info("Job " + jobId + " failed with state " + status.getState() + 
        " due to: " + status.getFailureInfo());
  }
  Counters counters = getCounters();
  if (counters != null) {
    LOG.info(counters.toString());
  }
  return success;
}
Example #19
Source File: ChainMapContextImpl.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Override
public IntegerRanges getProfileTaskRange(boolean isMap) {
  return base.getProfileTaskRange(isMap);
}
Example #20
Source File: ChainReduceContextImpl.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Override
public IntegerRanges getProfileTaskRange(boolean isMap) {
  return base.getProfileTaskRange(isMap);
}
Example #21
Source File: WrappedReducer.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Override
public IntegerRanges getProfileTaskRange(boolean isMap) {
  return reduceContext.getProfileTaskRange(isMap);
}
Example #22
Source File: WrappedMapper.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Override
public IntegerRanges getProfileTaskRange(boolean isMap) {
  return mapContext.getProfileTaskRange(isMap);
}
Example #23
Source File: Job.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
/**
 * Monitor a job and print status in real-time as progress is made and tasks 
 * fail.
 * @return true if the job succeeded
 * @throws IOException if communication to the JobTracker fails
 */
public boolean monitorAndPrintJob() 
    throws IOException, InterruptedException {
  String lastReport = null;
  Job.TaskStatusFilter filter;
  Configuration clientConf = getConfiguration();
  filter = Job.getTaskOutputFilter(clientConf);
  JobID jobId = getJobID();
  LOG.info("Running job: " + jobId);
  int eventCounter = 0;
  boolean profiling = getProfileEnabled();
  IntegerRanges mapRanges = getProfileTaskRange(true);
  IntegerRanges reduceRanges = getProfileTaskRange(false);
  int progMonitorPollIntervalMillis = 
    Job.getProgressPollInterval(clientConf);
  /* make sure to report full progress after the job is done */
  boolean reportedAfterCompletion = false;
  boolean reportedUberMode = false;
  while (!isComplete() || !reportedAfterCompletion) {
    if (isComplete()) {
      reportedAfterCompletion = true;
    } else {
      Thread.sleep(progMonitorPollIntervalMillis);
    }
    if (status.getState() == JobStatus.State.PREP) {
      continue;
    }      
    if (!reportedUberMode) {
      reportedUberMode = true;
      LOG.info("Job " + jobId + " running in uber mode : " + isUber());
    }      
    String report = 
      (" map " + StringUtils.formatPercent(mapProgress(), 0)+
          " reduce " + 
          StringUtils.formatPercent(reduceProgress(), 0));
    if (!report.equals(lastReport)) {
      LOG.info(report);
      lastReport = report;
    }

    TaskCompletionEvent[] events = 
      getTaskCompletionEvents(eventCounter, 10); 
    eventCounter += events.length;
    printTaskEvents(events, filter, profiling, mapRanges, reduceRanges);
  }
  boolean success = isSuccessful();
  if (success) {
    LOG.info("Job " + jobId + " completed successfully");
  } else {
    LOG.info("Job " + jobId + " failed with state " + status.getState() + 
        " due to: " + status.getFailureInfo());
  }
  Counters counters = getCounters();
  if (counters != null) {
    LOG.info(counters.toString());
  }
  return success;
}
Example #24
Source File: MockupMapContext.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
public IntegerRanges getProfileTaskRange(boolean isMap) {
    throw new NotImplementedException();
}
Example #25
Source File: HttpServer2.java    From hadoop-ozone with Apache License 2.0 4 votes vote down vote up 
public Builder setPortRanges(IntegerRanges ranges) {
  this.portRanges = ranges;
  return this;
}
Example #26
Source File: JobContextImpl.java    From big-c with Apache License 2.0 2 votes vote down vote up 
/**
 * Get the range of maps or reduces to profile.
 * @param isMap is the task a map?
 * @return the task ranges
 */
public IntegerRanges getProfileTaskRange(boolean isMap) {
  return conf.getProfileTaskRange(isMap);
}
Example #27
Source File: JobContext.java    From big-c with Apache License 2.0 2 votes vote down vote up 
/**
 * Get the range of maps or reduces to profile.
 * @param isMap is the task a map?
 * @return the task ranges
 */
public IntegerRanges getProfileTaskRange(boolean isMap);
Example #28
Source File: JobContext.java    From hadoop with Apache License 2.0 2 votes vote down vote up 
/**
 * Get the range of maps or reduces to profile.
 * @param isMap is the task a map?
 * @return the task ranges
 */
public IntegerRanges getProfileTaskRange(boolean isMap);
Example #29
Source File: JobContextImpl.java    From hadoop with Apache License 2.0 2 votes vote down vote up 
/**
 * Get the range of maps or reduces to profile.
 * @param isMap is the task a map?
 * @return the task ranges
 */
public IntegerRanges getProfileTaskRange(boolean isMap) {
  return conf.getProfileTaskRange(isMap);
}
Example #30
Source File: JobContextImpl.java    From incubator-tez with Apache License 2.0 2 votes vote down vote up 
/**
 * Get the range of maps or reduces to profile.
 * @param isMap is the task a map?
 * @return the task ranges
 */
public IntegerRanges getProfileTaskRange(boolean isMap) {
  return conf.getProfileTaskRange(isMap);
}