Java Code Examples for org.apache.tez.common.counters.TezCounter#getValue()

private float getAvgCounter(Collection<TaskAttemptInfo> taskAttemptInfos, String
    counterGroupName, String counterName) {
  long total = 0;
  int taskCount = 0;
  for (TaskAttemptInfo attemptInfo : taskAttemptInfos) {
    TezCounters tezCounters = attemptInfo.getTezCounters();
    TezCounter counter = tezCounters.findCounter(counterGroupName, counterName);
    if (counter != null) {
      total += counter.getValue();
      taskCount++;
    } else {
      LOG.info("Could not find counterGroupName=" + counterGroupName + ", counter=" +
          counterName + " in " + attemptInfo);
    }
  }
  return (taskCount > 0) ? (total * 1.0f / taskCount) : 0;
}
 

public void basicTest(int partitions, int numKeys, int keySize,
    long initialAvailableMem, int minBlockSize) throws IOException {
  this.numOutputs = partitions; // single output
  conf.setInt(TezRuntimeConfiguration
      .TEZ_RUNTIME_PIPELINED_SORTER_MIN_BLOCK_SIZE_IN_MB, minBlockSize >> 20);
  PipelinedSorter sorter = new PipelinedSorter(this.outputContext, conf, numOutputs,
      initialAvailableMem);

  writeData(sorter, numKeys, keySize);

  //partition stats;
  ReportPartitionStats partitionStats =
      ReportPartitionStats.fromString(conf.get(
      TezRuntimeConfiguration.TEZ_RUNTIME_REPORT_PARTITION_STATS,
      TezRuntimeConfiguration.TEZ_RUNTIME_REPORT_PARTITION_STATS_DEFAULT));
  if (partitionStats.isEnabled()) {
    assertTrue(sorter.getPartitionStats() != null);
  }

  verifyCounters(sorter, outputContext);
  verifyOutputPermissions(outputContext.getUniqueIdentifier());
  Path outputFile = sorter.finalOutputFile;
  FileSystem fs = outputFile.getFileSystem(conf);
  TezCounter finalOutputBytes =
      outputContext.getCounters().findCounter(TaskCounter.OUTPUT_BYTES_PHYSICAL);
  if (finalOutputBytes.getValue() > 0) {
    IFile.Reader reader = new IFile.Reader(fs, outputFile, null, null, null, false, -1, 4096);
    verifyData(reader);
    reader.close();
  }
  //Verify dataset
  verify(outputContext, atLeastOnce()).notifyProgress();
}
 

private int execute(String[] args, TezConfiguration tezConf, TezClient tezSession)
    throws IOException, TezException, InterruptedException {
  LOG.info("Running IntersectValidate");
  UserGroupInformation.setConfiguration(tezConf);

  String lhsDir = args[0];
  String rhsDir = args[1];
  int numPartitions = 1;
  if (args.length == 3) {
    numPartitions = Integer.parseInt(args[2]);
  }

  if (numPartitions <= 0) {
    System.err.println("NumPartitions must be > 0");
    return 4;
  }

  Path lhsPath = new Path(lhsDir);
  Path rhsPath = new Path(rhsDir);

  DAG dag = createDag(tezConf, lhsPath, rhsPath, numPartitions);
  setupURIsForCredentials(dag, lhsPath, rhsPath);

  tezSession.waitTillReady();
  DAGClient dagClient = tezSession.submitDAG(dag);
  DAGStatus dagStatus = dagClient.waitForCompletionWithAllStatusUpdates(null);
  if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {
    LOG.info("DAG diagnostics: " + dagStatus.getDiagnostics());
    return -1;
  } else {
    dagStatus = dagClient.getDAGStatus(Sets.newHashSet(StatusGetOpts.GET_COUNTERS));
    TezCounter counter = dagStatus.getDAGCounters().findCounter(COUNTER_GROUP_NAME,
        MISSING_KEY_COUNTER_NAME);
    if (counter == null) {
      LOG.info("Unable to determing equality");
      return -2;
    } else {
      if (counter.getValue() != 0) {
        LOG.info("Validate failed. The two sides are not equivalent");
        return -3;
      } else {
        LOG.info("Vlidation successful. The two sides are equivalent");
        return 0;
      }
    }
  }
}
 

@Override
protected int runJob(String[] args, TezConfiguration tezConf,
    TezClient tezClient) throws Exception {

  LOG.info("Running JoinValidate");

  String lhsDir = args[0];
  String rhsDir = args[1];
  int numPartitions = 1;
  if (args.length == 3) {
    numPartitions = Integer.parseInt(args[2]);
  }

  if (numPartitions <= 0) {
    System.err.println("NumPartitions must be > 0");
    return 4;
  }

  Path lhsPath = new Path(lhsDir);
  Path rhsPath = new Path(rhsDir);

  DAG dag = createDag(tezConf, lhsPath, rhsPath, numPartitions);

  tezClient.waitTillReady();
  DAGClient dagClient = tezClient.submitDAG(dag);
  Set<StatusGetOpts> getOpts = Sets.newHashSet();
  if (isCountersLog()) {
    getOpts.add(StatusGetOpts.GET_COUNTERS);
  }
  DAGStatus dagStatus = dagClient.waitForCompletionWithStatusUpdates(getOpts);
  if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {
    LOG.info("DAG diagnostics: " + dagStatus.getDiagnostics());
    return -1;
  } else {
    dagStatus = dagClient.getDAGStatus(Sets.newHashSet(StatusGetOpts.GET_COUNTERS));
    TezCounter counter = dagStatus.getDAGCounters().findCounter(COUNTER_GROUP_NAME,
        MISSING_KEY_COUNTER_NAME);
    if (counter == null) {
      LOG.info("Unable to determing equality");
      return -2;
    } else {
      if (counter.getValue() != 0) {
        LOG.info("Validate failed. The two sides are not equivalent");
        return -3;
      } else {
        LOG.info("Validation successful. The two sides are equivalent");
        return 0;
      }
    }
  }

}
 

@Test(timeout = 60000)
public void testPerIOCounterAggregation() throws Exception {
  String baseDir = "/tmp/perIOCounterAgg/";
  Path inPath1 = new Path(baseDir + "inPath1");
  Path inPath2 = new Path(baseDir + "inPath2");
  Path outPath = new Path(baseDir + "outPath");
  final Set<String> expectedResults = generateSortMergeJoinInput(inPath1, inPath2);
  Path stagingDirPath = new Path("/tmp/tez-staging-dir");
  remoteFs.mkdirs(stagingDirPath);

  TezConfiguration conf = new TezConfiguration(mrrTezCluster.getConfig());
  conf.setBoolean(TezConfiguration.TEZ_TASK_GENERATE_COUNTERS_PER_IO, true);
  TezClient tezClient = TezClient.create(SortMergeJoinHelper.class.getSimpleName(), conf);
  tezClient.start();

  SortMergeJoinHelper sortMergeJoinHelper = new SortMergeJoinHelper(tezClient);
  sortMergeJoinHelper.setConf(conf);

  String[] args = new String[] {
      "-D" + TezConfiguration.TEZ_AM_STAGING_DIR + "=" + stagingDirPath.toString(),
      "-counter", inPath1.toString(), inPath2.toString(), "1", outPath.toString() };
  assertEquals(0, sortMergeJoinHelper.run(conf, args, tezClient));

  verifySortMergeJoinInput(outPath, expectedResults);

  String joinerVertexName = "joiner";
  String input1Name = "input1";
  String input2Name = "input2";
  String joinOutputName = "joinOutput";
  Set<StatusGetOpts> statusOpts = new HashSet<StatusGetOpts>();
  statusOpts.add(StatusGetOpts.GET_COUNTERS);
  VertexStatus joinerVertexStatus =
      sortMergeJoinHelper.dagClient.getVertexStatus(joinerVertexName, statusOpts);
  final TezCounters joinerCounters = joinerVertexStatus.getVertexCounters();
  final CounterGroup aggregatedGroup = joinerCounters.getGroup(TaskCounter.class.getCanonicalName());
  final CounterGroup input1Group = joinerCounters.getGroup(
      TaskCounter.class.getSimpleName() + "_" + joinerVertexName + "_INPUT_" + input1Name);
  final CounterGroup input2Group = joinerCounters.getGroup(
      TaskCounter.class.getSimpleName() + "_" + joinerVertexName + "_INPUT_" + input2Name);
  assertTrue("aggregated counter group cannot be empty", aggregatedGroup.size() > 0);
  assertTrue("per io group for input1 cannot be empty", input1Group.size() > 0);
  assertTrue("per io group for input1 cannot be empty", input2Group.size() > 0);

  List<TaskCounter> countersToVerifyAgg = Arrays.asList(
      TaskCounter.ADDITIONAL_SPILLS_BYTES_READ,
      TaskCounter.ADDITIONAL_SPILLS_BYTES_WRITTEN,
      TaskCounter.COMBINE_INPUT_RECORDS,
      TaskCounter.MERGED_MAP_OUTPUTS,
      TaskCounter.NUM_DISK_TO_DISK_MERGES,
      TaskCounter.NUM_FAILED_SHUFFLE_INPUTS,
      TaskCounter.NUM_MEM_TO_DISK_MERGES,
      TaskCounter.NUM_SHUFFLED_INPUTS,
      TaskCounter.NUM_SKIPPED_INPUTS,
      TaskCounter.REDUCE_INPUT_GROUPS,
      TaskCounter.REDUCE_INPUT_RECORDS,
      TaskCounter.SHUFFLE_BYTES,
      TaskCounter.SHUFFLE_BYTES_DECOMPRESSED,
      TaskCounter.SHUFFLE_BYTES_DISK_DIRECT,
      TaskCounter.SHUFFLE_BYTES_TO_DISK,
      TaskCounter.SHUFFLE_BYTES_TO_MEM,
      TaskCounter.SPILLED_RECORDS
  );

  int nonZeroCounters = 0;
  // verify that the sum of the counter values for edges add up to the aggregated counter value.
  for(TaskCounter c : countersToVerifyAgg) {
    TezCounter aggregatedCounter = aggregatedGroup.findCounter(c.name(), false);
    TezCounter input1Counter = input1Group.findCounter(c.name(), false);
    TezCounter input2Counter = input2Group.findCounter(c.name(), false);
    assertNotNull("aggregated counter cannot be null " + c.name(), aggregatedCounter);
    assertNotNull("input1 counter cannot be null " + c.name(), input1Counter);
    assertNotNull("input2 counter cannot be null " + c.name(), input2Counter);

    assertEquals("aggregated counter does not match sum of input counters " + c.name(),
        aggregatedCounter.getValue(), input1Counter.getValue() + input2Counter.getValue());

    if (aggregatedCounter.getValue() > 0) {
      nonZeroCounters++;
    }
  }

  // ensure that at least one of the counters tested above were non-zero.
  assertTrue("At least one of the counter should be non-zero. invalid test ", nonZeroCounters > 0);

  CounterGroup joinerOutputGroup = joinerCounters.getGroup(
      TaskCounter.class.getSimpleName() + "_" + joinerVertexName + "_OUTPUT_" + joinOutputName);
  String outputCounterName = TaskCounter.OUTPUT_RECORDS.name();
  TezCounter aggregateCounter = aggregatedGroup.findCounter(outputCounterName, false);
  TezCounter joinerOutputCounter = joinerOutputGroup.findCounter(outputCounterName, false);
  assertNotNull("aggregated counter cannot be null " + outputCounterName, aggregateCounter);
  assertNotNull("output counter cannot be null " + outputCounterName, joinerOutputCounter);
  assertTrue("counter value is zero. test is invalid", aggregateCounter.getValue() > 0);
  assertEquals("aggregated counter does not match sum of output counters " + outputCounterName,
      aggregateCounter.getValue(), joinerOutputCounter.getValue());
}
 

public static Map<String,Object> convertCountersToATSMap(TezCounters counters) {
  Map<String, Object> object = new LinkedHashMap<String, Object>();
  if (counters == null) {
      return object;
    }
  ArrayList<Object> counterGroupsList = new ArrayList<Object>();
  for (CounterGroup group : counters) {
      ArrayList<Object> counterList = new ArrayList<Object>();
      for (TezCounter counter : group) {
        if (counter.getValue() != 0) {
          Map<String,Object> counterMap = new LinkedHashMap<String, Object>();
          counterMap.put(ATSConstants.COUNTER_NAME, counter.getName());
          if (!counter.getDisplayName().equals(counter.getName())) {
            counterMap.put(ATSConstants.COUNTER_DISPLAY_NAME,
                  counter.getDisplayName());
          }
          counterMap.put(ATSConstants.COUNTER_VALUE, counter.getValue());
          if (counter instanceof AggregateTezCounter) {
            counterMap.put(ATSConstants.COUNTER_INSTANCE_COUNT,
                ((AggregateTezCounter)counter).getCount());
            counterMap.put(ATSConstants.COUNTER_MAX_VALUE,
                  ((AggregateTezCounter)counter).getMax());
            counterMap.put(ATSConstants.COUNTER_MIN_VALUE,
                ((AggregateTezCounter)counter).getMin());

          }
          counterList.add(counterMap);
        }
      }
      if (!counterList.isEmpty()) {
        Map<String,Object> counterGroupMap = new LinkedHashMap<String, Object>();
        counterGroupMap.put(ATSConstants.COUNTER_GROUP_NAME, group.getName());
        if (!group.getDisplayName().equals(group.getName())) {
          counterGroupMap.put(ATSConstants.COUNTER_GROUP_DISPLAY_NAME,
              group.getDisplayName());
        }
        counterGroupMap.put(ATSConstants.COUNTERS, counterList);
        counterGroupsList.add(counterGroupMap);
      }
    }
  putInto(object, ATSConstants.COUNTER_GROUPS, counterGroupsList);
  return object;
}