Java Code Examples for org.apache.hadoop.mapreduce.CounterGroup#iterator()

The following examples show how to use org.apache.hadoop.mapreduce.CounterGroup#iterator() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: CounterDump.java    From datawave with Apache License 2.0 6 votes vote down vote up 
public String toString() {
    StringBuilder builder = new StringBuilder();
    
    while (source.hasNext()) {
        Entry<String,Counters> nextCntr = source.next();
        builder.append("\n").append(nextCntr.getKey()).append("\n----------------------\n");
        Counters counters = nextCntr.getValue();
        
        for (String groupName : counters.getGroupNames()) {
            
            CounterGroup group = counters.getGroup(groupName);
            Iterator<Counter> cntrItr = group.iterator();
            while (cntrItr.hasNext()) {
                Counter counter = cntrItr.next();
                builder.append(groupName).append("\t").append(counter.getDisplayName()).append("=").append(counter.getValue()).append("\n");
            }
            
        }
    }
    
    return builder.toString();
}
Example 2
Source File: JoinPhaseJob.java    From incubator-pinot with Apache License 2.0 6 votes vote down vote up 
private void dumpSummary(Job job, List<String> sourceNames) throws IOException {
  System.out.println("Join Input Matrix.");
  CounterGroup group = job.getCounters().getGroup("DynamicCounter");
  for (String source : sourceNames) {
    System.out.print(String.format("%25s\t", source));
  }
  if (group != null) {
    Iterator<Counter> iterator = group.iterator();
    while (iterator.hasNext()) {
      Counter counter = iterator.next();
      String displayName = counter.getDisplayName();
      String[] split = displayName.replace("[", "").replace("[", "").split(",");
      for (String str : split) {
        if (str.trim().equals("1")) {
          System.out.print(String.format("%25s\t", "1"));
        } else {
          System.out.print(String.format("%25s\t", "-"));
        }
      }
    }
  }
}
Example 3
Source File: HistoryViewer.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
private void printCounters(StringBuffer buff, Counters totalCounters,
    Counters mapCounters, Counters reduceCounters) {
  // Killed jobs might not have counters
  if (totalCounters == null) {
    return;
  }
  buff.append("\nCounters: \n\n");
  buff.append(String.format("|%1$-30s|%2$-30s|%3$-10s|%4$-10s|%5$-10s|", 
      "Group Name",
      "Counter name",
      "Map Value",
      "Reduce Value",
      "Total Value"));
  buff.append("\n------------------------------------------"+
      "---------------------------------------------");
  for (String groupName : totalCounters.getGroupNames()) {
       CounterGroup totalGroup = totalCounters.getGroup(groupName);
       CounterGroup mapGroup = mapCounters.getGroup(groupName);
       CounterGroup reduceGroup = reduceCounters.getGroup(groupName);
    
       Format decimal = new DecimalFormat();
       Iterator<org.apache.hadoop.mapreduce.Counter> ctrItr =
         totalGroup.iterator();
       while(ctrItr.hasNext()) {
         org.apache.hadoop.mapreduce.Counter counter = ctrItr.next();
         String name = counter.getName();
         String mapValue = 
           decimal.format(mapGroup.findCounter(name).getValue());
         String reduceValue = 
           decimal.format(reduceGroup.findCounter(name).getValue());
         String totalValue = 
           decimal.format(counter.getValue());

         buff.append(
             String.format("%n|%1$-30s|%2$-30s|%3$-10s|%4$-10s|%5$-10s", 
                 totalGroup.getDisplayName(),
                 counter.getDisplayName(),
                 mapValue, reduceValue, totalValue));
    }
  }
}
Example 4
Source File: HistoryViewer.java    From big-c with Apache License 2.0 4 votes vote down vote up 
private void printCounters(StringBuffer buff, Counters totalCounters,
    Counters mapCounters, Counters reduceCounters) {
  // Killed jobs might not have counters
  if (totalCounters == null) {
    return;
  }
  buff.append("\nCounters: \n\n");
  buff.append(String.format("|%1$-30s|%2$-30s|%3$-10s|%4$-10s|%5$-10s|", 
      "Group Name",
      "Counter name",
      "Map Value",
      "Reduce Value",
      "Total Value"));
  buff.append("\n------------------------------------------"+
      "---------------------------------------------");
  for (String groupName : totalCounters.getGroupNames()) {
       CounterGroup totalGroup = totalCounters.getGroup(groupName);
       CounterGroup mapGroup = mapCounters.getGroup(groupName);
       CounterGroup reduceGroup = reduceCounters.getGroup(groupName);
    
       Format decimal = new DecimalFormat();
       Iterator<org.apache.hadoop.mapreduce.Counter> ctrItr =
         totalGroup.iterator();
       while(ctrItr.hasNext()) {
         org.apache.hadoop.mapreduce.Counter counter = ctrItr.next();
         String name = counter.getName();
         String mapValue = 
           decimal.format(mapGroup.findCounter(name).getValue());
         String reduceValue = 
           decimal.format(reduceGroup.findCounter(name).getValue());
         String totalValue = 
           decimal.format(counter.getValue());

         buff.append(
             String.format("%n|%1$-30s|%2$-30s|%3$-10s|%4$-10s|%5$-10s", 
                 totalGroup.getDisplayName(),
                 counter.getDisplayName(),
                 mapValue, reduceValue, totalValue));
    }
  }
}
Example 5
Source File: Main.java    From hiped2 with Apache License 2.0 4 votes vote down vote up 
public static boolean findShortestPath(Configuration conf, Path inputPath,
                                       Path outputPath, String startNode,
                                       String targetNode)
    throws Exception {
  conf.set(TARGET_NODE, targetNode);

  Job job = new Job(conf);
  job.setJarByClass(Main.class);
  job.setMapperClass(Map.class);
  job.setReducerClass(Reduce.class);

  job.setInputFormatClass(KeyValueTextInputFormat.class);

  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(Text.class);

  FileInputFormat.setInputPaths(job, inputPath);
  FileOutputFormat.setOutputPath(job, outputPath);

  if (!job.waitForCompletion(true)) {
    throw new Exception("Job failed");
  }

  Counter counter = job.getCounters()
      .findCounter(Reduce.PathCounter.TARGET_NODE_DISTANCE_COMPUTED);

  if (counter != null && counter.getValue() > 0) {
    CounterGroup group = job.getCounters().getGroup(Reduce.PathCounter.PATH.toString());
    Iterator<Counter> iter = group.iterator();
    iter.hasNext();
    String path = iter.next().getName();
    System.out.println("==========================================");
    System.out.println("= Shortest path found, details as follows.");
    System.out.println("= ");
    System.out.println("= Start node:  " + startNode);
    System.out.println("= End node:    " + targetNode);
    System.out.println("= Hops:        " + counter.getValue());
    System.out.println("= Path:        " + path);
    System.out.println("==========================================");
    return true;
  }
  return false;
}