org.apache.hadoop.mapred.Counters Java Examples

private void combineAndSpill(
    RawKeyValueIterator kvIter,
    Counters.Counter inCounter) throws IOException {
  JobConf job = jobConf;
  Reducer combiner = ReflectionUtils.newInstance(combinerClass, job);
  Class<K> keyClass = (Class<K>) job.getMapOutputKeyClass();
  Class<V> valClass = (Class<V>) job.getMapOutputValueClass();
  RawComparator<K> comparator = 
    (RawComparator<K>)job.getCombinerKeyGroupingComparator();
  try {
    CombineValuesIterator values = new CombineValuesIterator(
        kvIter, comparator, keyClass, valClass, job, Reporter.NULL,
        inCounter);
    while (values.more()) {
      combiner.reduce(values.getKey(), values, combineCollector,
                      Reporter.NULL);
      values.nextKey();
    }
  } finally {
    combiner.close();
  }
}
 

@Nonnull
private static Booster train(@Nonnull final DMatrix dtrain, @Nonnegative final int round,
        @Nonnull final Map<String, Object> params, @Nullable final Reporter reporter)
        throws NoSuchMethodException, IllegalAccessException, InvocationTargetException,
        InstantiationException, XGBoostError {
    final Counters.Counter iterCounter = (reporter == null) ? null
            : reporter.getCounter("hivemall.XGBoostTrainUDTF$Counter", "iteration");

    final Booster booster = XGBoostUtils.createBooster(dtrain, params);
    for (int iter = 0; iter < round; iter++) {
        reportProgress(reporter);
        setCounterValue(iterCounter, iter + 1);

        booster.update(dtrain, iter);
    }
    return booster;
}
 

@Before
@SuppressWarnings("unchecked") // mocked generics
public void setup() {
  LOG.info(">>>> " + name.getMethodName());
  job = new JobConf();
  job.setBoolean(MRJobConfig.SHUFFLE_FETCH_RETRY_ENABLED, false);
  jobWithRetry = new JobConf();
  jobWithRetry.setBoolean(MRJobConfig.SHUFFLE_FETCH_RETRY_ENABLED, true);
  id = TaskAttemptID.forName("attempt_0_1_r_1_1");
  ss = mock(ShuffleSchedulerImpl.class);
  mm = mock(MergeManagerImpl.class);
  r = mock(Reporter.class);
  metrics = mock(ShuffleClientMetrics.class);
  except = mock(ExceptionReporter.class);
  key = JobTokenSecretManager.createSecretKey(new byte[]{0,0,0,0});
  connection = mock(HttpURLConnection.class);

  allErrs = mock(Counters.Counter.class);
  when(r.getCounter(anyString(), anyString())).thenReturn(allErrs);

  ArrayList<TaskAttemptID> maps = new ArrayList<TaskAttemptID>(1);
  maps.add(map1ID);
  maps.add(map2ID);
  when(ss.getMapsForHost(host)).thenReturn(maps);
}
 

private void combineAndSpill(
    RawKeyValueIterator kvIter,
    Counters.Counter inCounter) throws IOException {
  JobConf job = jobConf;
  Reducer combiner = ReflectionUtils.newInstance(combinerClass, job);
  Class<K> keyClass = (Class<K>) job.getMapOutputKeyClass();
  Class<V> valClass = (Class<V>) job.getMapOutputValueClass();
  RawComparator<K> comparator = 
    (RawComparator<K>)job.getCombinerKeyGroupingComparator();
  try {
    CombineValuesIterator values = new CombineValuesIterator(
        kvIter, comparator, keyClass, valClass, job, Reporter.NULL,
        inCounter);
    while (values.more()) {
      combiner.reduce(values.getKey(), values, combineCollector,
                      Reporter.NULL);
      values.nextKey();
    }
  } finally {
    combiner.close();
  }
}
 

@Before
@SuppressWarnings("unchecked") // mocked generics
public void setup() {
  LOG.info(">>>> " + name.getMethodName());
  job = new JobConf();
  job.setBoolean(MRJobConfig.SHUFFLE_FETCH_RETRY_ENABLED, false);
  jobWithRetry = new JobConf();
  jobWithRetry.setBoolean(MRJobConfig.SHUFFLE_FETCH_RETRY_ENABLED, true);
  id = TaskAttemptID.forName("attempt_0_1_r_1_1");
  ss = mock(ShuffleSchedulerImpl.class);
  mm = mock(MergeManagerImpl.class);
  r = mock(Reporter.class);
  metrics = mock(ShuffleClientMetrics.class);
  except = mock(ExceptionReporter.class);
  key = JobTokenSecretManager.createSecretKey(new byte[]{0,0,0,0});
  connection = mock(HttpURLConnection.class);

  allErrs = mock(Counters.Counter.class);
  when(r.getCounter(anyString(), anyString())).thenReturn(allErrs);

  ArrayList<TaskAttemptID> maps = new ArrayList<TaskAttemptID>(1);
  maps.add(map1ID);
  maps.add(map2ID);
  when(ss.getMapsForHost(host)).thenReturn(maps);
}
 

public static String extractCounter(String counterFromHist,
                                    String... counterNames)
    throws ParseException {
  Counters counters =
      Counters.fromEscapedCompactString(counterFromHist);
  for (Counters.Group group : counters) {
    for (Counters.Counter counter : group) {
      for (String counterName : counterNames) {
        if (counterName.equals(counter.getName())) {
          return String.valueOf(counter.getCounter());
        }
      }
    }
  }
  return null;
}
 

public static String extractCounter(String counterFromHist,
                                    String... counterNames)
    throws ParseException {
  Counters counters =
      Counters.fromEscapedCompactString(counterFromHist);
  for (Counters.Group group : counters) {
    for (Counters.Counter counter : group) {
      for (String counterName : counterNames) {
        if (counterName.equals(counter.getName())) {
          return String.valueOf(counter.getCounter());
        }
      }
    }
  }
  return "";
}
 

/**
 * Returns the count for the given counter name in the counter group
 * 'MultiStoreCounters'
 *
 * @param job the MR job
 * @param jobClient the Hadoop job client
 * @param counterName the counter name
 * @return the count of the given counter name
 */
public static long getMultiStoreCount(Job job, JobClient jobClient,
        String counterName) {
    long value = -1;
    try {
        RunningJob rj = jobClient.getJob(job.getAssignedJobID());
        if (rj != null) {
            Counters.Counter counter = rj.getCounters().getGroup(
                    MULTI_STORE_COUNTER_GROUP).getCounterForName(counterName);
            value = counter.getValue();
        }
    } catch (IOException e) {
        LOG.warn("Failed to get the counter for " + counterName, e);
    }
    return value;
}
 

public ShuffleSchedulerImpl(JobConf job, TaskStatus status,
                        TaskAttemptID reduceId,
                        ExceptionReporter reporter,
                        Progress progress,
                        Counters.Counter shuffledMapsCounter,
                        Counters.Counter reduceShuffleBytes,
                        Counters.Counter failedShuffleCounter) {
  totalMaps = job.getNumMapTasks();
  abortFailureLimit = Math.max(30, totalMaps / 10);
  copyTimeTracker = new CopyTimeTracker();
  remainingMaps = totalMaps;
  finishedMaps = new boolean[remainingMaps];
  this.reporter = reporter;
  this.status = status;
  this.reduceId = reduceId;
  this.progress = progress;
  this.shuffledMapsCounter = shuffledMapsCounter;
  this.reduceShuffleBytes = reduceShuffleBytes;
  this.failedShuffleCounter = failedShuffleCounter;
  this.startTime = Time.monotonicNow();
  lastProgressTime = startTime;
  referee.start();
  this.maxFailedUniqueFetches = Math.min(totalMaps, 5);
  this.maxFetchFailuresBeforeReporting = job.getInt(
      MRJobConfig.SHUFFLE_FETCH_FAILURES, REPORT_FAILURE_LIMIT);
  this.reportReadErrorImmediately = job.getBoolean(
      MRJobConfig.SHUFFLE_NOTIFY_READERROR, true);

  this.maxDelay = job.getLong(MRJobConfig.MAX_SHUFFLE_FETCH_RETRY_DELAY,
      MRJobConfig.DEFAULT_MAX_SHUFFLE_FETCH_RETRY_DELAY);
  this.maxHostFailures = job.getInt(
      MRJobConfig.MAX_SHUFFLE_FETCH_HOST_FAILURES,
      MRJobConfig.DEFAULT_MAX_SHUFFLE_FETCH_HOST_FAILURES);
}
 

private void validateCounters() throws IOException {
  Counters counters = job.running_.getCounters();
  assertNotNull("Counters", counters);
  Group group = counters.getGroup("UserCounters");
  assertNotNull("Group", group);
  Counter counter = group.getCounterForName("InputLines");
  assertNotNull("Counter", counter);
  assertEquals(3, counter.getCounter());
}
 

@Test
public void testCommandLine() throws Exception {
  super.testCommandLine();
  // validate combiner counters
  String counterGrp = "org.apache.hadoop.mapred.Task$Counter";
  Counters counters = job.running_.getCounters();
  assertTrue(counters.findCounter(
             counterGrp, "COMBINE_INPUT_RECORDS").getValue() != 0);
  assertTrue(counters.findCounter(
             counterGrp, "COMBINE_OUTPUT_RECORDS").getValue() != 0);
}
 

private void verifyLaunchedTasks(SleepJob sleepJob, int maps, int reduces)
    throws IOException {
  Counters jobCounters = sleepJob.getRunningJob().getCounters();
  long launchedMaps = jobCounters.findCounter(
      JobInProgress.Counter.TOTAL_LAUNCHED_MAPS).getValue();
  long launchedReduces = jobCounters.findCounter(
      JobInProgress.Counter.TOTAL_LAUNCHED_REDUCES).getValue();
  Assert.assertEquals(maps, launchedMaps);
  Assert.assertEquals(reduces, launchedReduces);
}
 

public void incrementCounter(int id, long amount) throws IOException {
  if (id < registeredCounters.size()) {
    Counters.Counter counter = registeredCounters.get(id);
    counter.increment(amount);
  } else {
    throw new IOException("Invalid counter with id: " + id);
  }
}
 

private String toRaidProgressHtmlRow(String dateStr, Counters ctrs) {
  StringBuilder sb = new StringBuilder();
  sb.append(td(dateStr));
  sb.append(td(Long.toString(ctrs.getCounter(Counter.FILES_SUCCEEDED))));
  sb.append(td(StringUtils.humanReadableInt(ctrs.getCounter(Counter.PROCESSED_SIZE))));
  sb.append(td(StringUtils.humanReadableInt(ctrs.getCounter(Counter.SAVING_SIZE))));
  return tr(sb.toString());
}
 

public ShuffleSchedulerImpl(JobConf job, TaskStatus status,
                        TaskAttemptID reduceId,
                        ExceptionReporter reporter,
                        Progress progress,
                        Counters.Counter shuffledMapsCounter,
                        Counters.Counter reduceShuffleBytes,
                        Counters.Counter failedShuffleCounter) {
  totalMaps = job.getNumMapTasks();
  abortFailureLimit = Math.max(30, totalMaps / 10);
  copyTimeTracker = new CopyTimeTracker();
  remainingMaps = totalMaps;
  finishedMaps = new boolean[remainingMaps];
  this.reporter = reporter;
  this.status = status;
  this.reduceId = reduceId;
  this.progress = progress;
  this.shuffledMapsCounter = shuffledMapsCounter;
  this.reduceShuffleBytes = reduceShuffleBytes;
  this.failedShuffleCounter = failedShuffleCounter;
  this.startTime = Time.monotonicNow();
  lastProgressTime = startTime;
  referee.start();
  this.maxFailedUniqueFetches = Math.min(totalMaps, 5);
  this.maxFetchFailuresBeforeReporting = job.getInt(
      MRJobConfig.SHUFFLE_FETCH_FAILURES, REPORT_FAILURE_LIMIT);
  this.reportReadErrorImmediately = job.getBoolean(
      MRJobConfig.SHUFFLE_NOTIFY_READERROR, true);

  this.maxDelay = job.getLong(MRJobConfig.MAX_SHUFFLE_FETCH_RETRY_DELAY,
      MRJobConfig.DEFAULT_MAX_SHUFFLE_FETCH_RETRY_DELAY);
  this.maxHostFailures = job.getInt(
      MRJobConfig.MAX_SHUFFLE_FETCH_HOST_FAILURES,
      MRJobConfig.DEFAULT_MAX_SHUFFLE_FETCH_HOST_FAILURES);
}
 

private Counters covertToHadoopCounters(TezCounters tezCounters) {
    Counters counters = new Counters();
    for (CounterGroup tezGrp : tezCounters) {
        Group grp = counters.addGroup(tezGrp.getName(), tezGrp.getDisplayName());
        for (TezCounter counter : tezGrp) {
            grp.addCounter(counter.getName(), counter.getDisplayName(), counter.getValue());
        }
    }
    return counters;
}
 

private void validateCounters() throws IOException {
  Counters counters = job.running_.getCounters();
  assertNotNull("Counters", counters);
  Group group = counters.getGroup("UserCounters");
  assertNotNull("Group", group);
  Counter counter = group.getCounterForName("InputLines");
  assertNotNull("Counter", counter);
  assertEquals(3, counter.getCounter());
}
 

@Test
public void testCommandLine() throws Exception {
  super.testCommandLine();
  // validate combiner counters
  String counterGrp = "org.apache.hadoop.mapred.Task$Counter";
  Counters counters = job.running_.getCounters();
  assertTrue(counters.findCounter(
             counterGrp, "COMBINE_INPUT_RECORDS").getValue() != 0);
  assertTrue(counters.findCounter(
             counterGrp, "COMBINE_OUTPUT_RECORDS").getValue() != 0);
}
 

public Counters.Counter getCounter(String group, String name) {
  TezCounter counter = context.getCounters().findCounter(group, name);
  MRCounters.MRCounter mrCounter = null;
  if (counter != null) {
    mrCounter = new MRCounters.MRCounter(counter);
  }
  return mrCounter;
}
 

private void assertAllDocumentsOk(String script, Configuration conf) throws Exception {
    PigServer ps = setup(script, conf);
    List<ExecJob> jobs = ps.executeBatch();
    PigStats stats = jobs.get(0).getStatistics();
    for (JobStats js : stats.getJobGraph()) {
        Counters hadoopCounters = ((MRJobStats)js).getHadoopCounters();
        assertNotNull(hadoopCounters);
        VespaCounters counters = VespaCounters.get(hadoopCounters);
        assertEquals(10, counters.getDocumentsSent());
        assertEquals(0, counters.getDocumentsFailed());
        assertEquals(10, counters.getDocumentsOk());
    }
}
 

@Override
public Counter getCounter(String group, String name)
{
  Counters.Counter counter = null;
  if (counters != null) {
    counter = counters.findCounter(group, name);
  }

  return counter;
}
 

@Override
public Counter getCounter(Enum<?> key)
{
  Counters.Counter counter = null;
  if (counters != null) {
    counter = counters.findCounter(key);
  }

  return counter;
}
 

@Override
public void setup(OperatorContext context)
{
  reporter = new ReporterImpl(ReporterImpl.ReporterType.Reducer, new Counters());
  if (context != null) {
    operatorId = context.getId();
  }
  cacheObject = new HashMap<K1, List<V1>>();
  outputCollector = new OutputCollectorImpl<K2, V2>();
  if (reduceClass != null) {
    try {
      reduceObj = reduceClass.newInstance();
    } catch (Exception e) {
      logger.info("can't instantiate object {}", e.getMessage());
      throw new RuntimeException(e);
    }
    Configuration conf = new Configuration();
    InputStream stream = null;
    if (configFile != null && configFile.length() > 0) {
      logger.info("system /{}", configFile);
      stream = ClassLoader.getSystemResourceAsStream("/" + configFile);
      if (stream == null) {
        logger.info("system {}", configFile);
        stream = ClassLoader.getSystemResourceAsStream(configFile);
      }
    }
    if (stream != null) {
      logger.info("found our stream... so adding it");
      conf.addResource(stream);
    }
    reduceObj.configure(new JobConf(conf));
  }

}
 

public static Counters getCounters(Job job) throws IOException {
    try {
        Cluster cluster = new Cluster(job.getJobConf());
        org.apache.hadoop.mapreduce.Job mrJob = cluster.getJob(job.getAssignedJobID());
        if (mrJob == null) { // In local mode, mrJob will be null
            mrJob = job.getJob();
        }
        return new Counters(mrJob.getCounters());
    } catch (Exception ir) {
        throw new IOException(ir);
    }
}
 

@SuppressWarnings("deprecation")
private static void parseAndAddJobCounters(Map<String, String> job, String counters) {
    try {
        Counters counterGroups = Counters.fromEscapedCompactString(counters);
        for (Group otherGroup : counterGroups) {
            Group group = counterGroups.getGroup(otherGroup.getName());
            for (Counter otherCounter : otherGroup) {
                Counter counter = group.getCounterForName(otherCounter.getName());
                job.put(otherCounter.getName(), String.valueOf(counter.getValue()));
            }
        }
    } catch (ParseException e) {
       LOG.warn("Failed to parse job counters", e);
    }
}
 

@Override
public List<HadoopCounterKeyValuePair> getAllCounters() throws IOException {
	List<HadoopCounterKeyValuePair> result = new ArrayList<HadoopCounterKeyValuePair>();
	for (Counters.Group group : job.getCounters()) {
		for (Counter counter : group) {
			result.add(new HadoopCounterKeyValuePair(counter.getName(), group.getName(), counter.getValue()));
		}
	}
	return result;
}
 

@BeforeClass
public void setUp() throws Exception {

  String contextName = CONTEXT_NAME + "_" + UUID.randomUUID().toString();

  Reporter mockedReporter = Mockito.mock(Reporter.class);

  this.recordsProcessedCount = Mockito.mock(Counters.Counter.class);
  Mockito.when(mockedReporter.getCounter(
      contextName, MetricRegistry.name(RECORDS_PROCESSED, Measurements.COUNT.getName())))
      .thenReturn(this.recordsProcessedCount);

  this.recordProcessRateCount = Mockito.mock(Counters.Counter.class);
  Mockito.when(mockedReporter.getCounter(
      contextName, MetricRegistry.name(RECORD_PROCESS_RATE, Measurements.COUNT.getName())))
      .thenReturn(this.recordProcessRateCount);

  this.recordSizeDistributionCount = Mockito.mock(Counters.Counter.class);
  Mockito.when(mockedReporter.getCounter(
      contextName, MetricRegistry.name(RECORD_SIZE_DISTRIBUTION, Measurements.COUNT.getName())))
      .thenReturn(this.recordSizeDistributionCount);

  this.totalDurationCount = Mockito.mock(Counters.Counter.class);
  Mockito.when(mockedReporter.getCounter(
      contextName, MetricRegistry.name(TOTAL_DURATION, Measurements.COUNT.getName())))
      .thenReturn(this.totalDurationCount);

  this.queueSize = Mockito.mock(Counters.Counter.class);
  Mockito.when(mockedReporter.getCounter(contextName, QUEUE_SIZE)).thenReturn(this.queueSize);

  this.hadoopCounterReporter = HadoopCounterReporter.builder(mockedReporter)
      .convertRatesTo(TimeUnit.SECONDS)
      .convertDurationsTo(TimeUnit.SECONDS)
      .filter(MetricFilter.ALL)
      .build(MetricContext.builder(contextName).buildStrict());
}
 

public Counters.Counter getCounter(Enum<?> name) {
  TezCounter counter = context.getCounters().findCounter(name);
  MRCounters.MRCounter mrCounter = null;
  if (counter != null) {
    mrCounter = new MRCounters.MRCounter(counter);
  }
  return mrCounter;
}
 

@Override
public ICounter getCounterByNameAndFlag(String groupName, String counterName, String counterFlag) {
  if (conf.getBoolean(counterFlag, true)) {
    Counters.Counter counter = reporter.getCounter(groupName, counterName);
    if (counter != null) {
      return new MapRedCounterAdapter(reporter.getCounter(groupName, counterName));
    }
  }
  return new BenchmarkCounter.NullCounter();
  }
 

/**
 * Print this job counters (for debugging purpose)
 */
void printCounters() {
  System.out.printf("New Job:\n", counters);
  for (String groupName : counters.getGroupNames()) {
    Counters.Group group = counters.getGroup(groupName);
    System.out.printf("\t%s[%s]\n", groupName, group.getDisplayName());

    for (Counters.Counter counter : group) {
      System.out.printf("\t\t%s: %s\n", counter.getDisplayName(),
                                        counter.getCounter());
    }
  }
  System.out.printf("\n");
}