Java Code Examples for org.apache.tez.common.counters.TezCounter#increment()

The following examples show how to use org.apache.tez.common.counters.TezCounter#increment() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TezTaskContext.java    From spork with Apache License 2.0 5 votes vote down vote up 
@Override
public boolean incrCounter(Enum<?> name, long delta) {
    if (context == null) {
        return false;
    }
    TezCounter counter = context.getCounters().findCounter(name);
    counter.increment(delta);
    return true;
}
Example 2
Source File: TezTaskContext.java    From spork with Apache License 2.0 5 votes vote down vote up 
@Override
public boolean incrCounter(String group, String name, long delta) {
    if (context == null) {
        return false;
    }
    TezCounter counter = context.getCounters().getGroup(group).findCounter(name);
    counter.increment(delta);
    return true;
}
Example 3
Source File: IFile.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up 
/**
 * Construct an IFile Reader.
 *
 * @param in   The input stream
 * @param length Length of the data in the stream, including the checksum
 *               bytes.
 * @param codec codec
 * @param readsCounter Counter for records read from disk
 * @throws IOException
 */
public Reader(InputStream in, long length,
    CompressionCodec codec,
    TezCounter readsCounter, TezCounter bytesReadCounter,
    boolean readAhead, int readAheadLength,
    int bufferSize) throws IOException {
  this(in, ((in != null) ? (length - HEADER.length) : length), codec,
      readsCounter, bytesReadCounter, readAhead, readAheadLength,
      bufferSize, ((in != null) ? isCompressedFlagEnabled(in) : false));
  if (in != null && bytesReadCounter != null) {
    bytesReadCounter.increment(IFile.HEADER.length);
  }
}
Example 4
Source File: IntersectValidate.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up 
@Override
public void run() throws Exception {
  Preconditions.checkState(getInputs().size() == 2);
  Preconditions.checkState(getOutputs().size() == 0);
  LogicalInput lhsInput = getInputs().get(LHS_INPUT_NAME);
  LogicalInput rhsInput = getInputs().get(RHS_INPUT_NAME);
  Reader lhsReaderRaw = lhsInput.getReader();
  Reader rhsReaderRaw = rhsInput.getReader();
  Preconditions.checkState(lhsReaderRaw instanceof KeyValuesReader);
  Preconditions.checkState(rhsReaderRaw instanceof KeyValuesReader);
  KeyValuesReader lhsReader = (KeyValuesReader) lhsReaderRaw;
  KeyValuesReader rhsReader = (KeyValuesReader) rhsReaderRaw;

  TezCounter lhsMissingKeyCounter = getContext().getCounters().findCounter(COUNTER_GROUP_NAME,
      MISSING_KEY_COUNTER_NAME);

  while (lhsReader.next()) {
    if (rhsReader.next()) {
      if (!lhsReader.getCurrentKey().equals(rhsReader.getCurrentKey())) {
        LOG.info("MismatchedKeys: " + "lhs=" + lhsReader.getCurrentKey() + ", rhs=" + rhsReader.getCurrentKey());
        lhsMissingKeyCounter.increment(1);
      }
    } else {
      lhsMissingKeyCounter.increment(1);
      LOG.info("ExtraKey in lhs: " + lhsReader.getClass());
      break;
    }
  }
  if (rhsReader.next()) {
    lhsMissingKeyCounter.increment(1);
    LOG.info("ExtraKey in rhs: " + lhsReader.getClass());
  }
}
Example 5
Source File: MRInputUtils.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
public static org.apache.hadoop.mapreduce.InputSplit getNewSplitDetailsFromDisk(
    TaskSplitIndex splitMetaInfo, JobConf jobConf, TezCounter splitBytesCounter)
    throws IOException {
  Path file = new Path(splitMetaInfo.getSplitLocation());
  long offset = splitMetaInfo.getStartOffset();

  // Split information read from local filesystem.
  FileSystem fs = FileSystem.getLocal(jobConf);
  file = fs.makeQualified(file);
  LOG.info("Reading input split file from : " + file);
  FSDataInputStream inFile = fs.open(file);
  inFile.seek(offset);
  String className = Text.readString(inFile);
  Class<org.apache.hadoop.mapreduce.InputSplit> cls;
  try {
    cls = (Class<org.apache.hadoop.mapreduce.InputSplit>) jobConf.getClassByName(className);
  } catch (ClassNotFoundException ce) {
    IOException wrap = new IOException("Split class " + className + " not found");
    wrap.initCause(ce);
    throw wrap;
  }
  SerializationFactory factory = new SerializationFactory(jobConf);
  Deserializer<org.apache.hadoop.mapreduce.InputSplit> deserializer = (Deserializer<org.apache.hadoop.mapreduce.InputSplit>) factory
      .getDeserializer(cls);
  deserializer.open(inFile);
  org.apache.hadoop.mapreduce.InputSplit split = deserializer.deserialize(null);
  long pos = inFile.getPos();
  if (splitBytesCounter != null) {
    splitBytesCounter.increment(pos - offset);
  }
  inFile.close();
  return split;
}
Example 6
Source File: MRInputUtils.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
public static InputSplit getOldSplitDetailsFromDisk(TaskSplitIndex splitMetaInfo,
    JobConf jobConf, TezCounter splitBytesCounter) throws IOException {
  Path file = new Path(splitMetaInfo.getSplitLocation());
  FileSystem fs = FileSystem.getLocal(jobConf);
  file = fs.makeQualified(file);
  LOG.info("Reading input split file from : " + file);
  long offset = splitMetaInfo.getStartOffset();

  FSDataInputStream inFile = fs.open(file);
  inFile.seek(offset);
  String className = Text.readString(inFile);
  Class<org.apache.hadoop.mapred.InputSplit> cls;
  try {
    cls = (Class<org.apache.hadoop.mapred.InputSplit>) jobConf.getClassByName(className);
  } catch (ClassNotFoundException ce) {
    IOException wrap = new IOException("Split class " + className + " not found");
    wrap.initCause(ce);
    throw wrap;
  }
  SerializationFactory factory = new SerializationFactory(jobConf);
  Deserializer<org.apache.hadoop.mapred.InputSplit> deserializer = (Deserializer<org.apache.hadoop.mapred.InputSplit>) factory
      .getDeserializer(cls);
  deserializer.open(inFile);
  org.apache.hadoop.mapred.InputSplit split = deserializer.deserialize(null);
  long pos = inFile.getPos();
  if (splitBytesCounter != null) {
    splitBytesCounter.increment(pos - offset);
  }
  inFile.close();
  return split;
}
Example 7
Source File: GcTimeUpdater.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up 
/**
 * Increment the gc-elapsed-time counter.
 */
void incrementGcCounter() {
  if (null == counters) {
    return; // nothing to do.
  }

  TezCounter gcCounter = counters.findCounter(TaskCounter.GC_TIME_MILLIS);
  if (null != gcCounter) {
    gcCounter.increment(getElapsedGc());
  }
}
Example 8
Source File: JoinValidate.java    From tez with Apache License 2.0 5 votes vote down vote up 
@Override
public void run() throws Exception {
  Preconditions.checkState(getInputs().size() == 2);
  Preconditions.checkState(getOutputs().size() == 0);
  LogicalInput lhsInput = getInputs().get(LHS_INPUT_NAME);
  LogicalInput rhsInput = getInputs().get(RHS_INPUT_NAME);
  Reader lhsReaderRaw = lhsInput.getReader();
  Reader rhsReaderRaw = rhsInput.getReader();
  Preconditions.checkState(lhsReaderRaw instanceof KeyValuesReader);
  Preconditions.checkState(rhsReaderRaw instanceof KeyValuesReader);
  KeyValuesReader lhsReader = (KeyValuesReader) lhsReaderRaw;
  KeyValuesReader rhsReader = (KeyValuesReader) rhsReaderRaw;
  boolean rhsReaderEnd = false;

  TezCounter lhsMissingKeyCounter = getContext().getCounters().findCounter(COUNTER_GROUP_NAME,
      MISSING_KEY_COUNTER_NAME);

  while (lhsReader.next()) {
    if (rhsReader.next()) {
      if (!lhsReader.getCurrentKey().equals(rhsReader.getCurrentKey())) {
        LOG.info("MismatchedKeys: " + "lhs=" + lhsReader.getCurrentKey() + ", rhs=" + rhsReader.getCurrentKey());
        lhsMissingKeyCounter.increment(1);
      }
    } else {
      lhsMissingKeyCounter.increment(1);
      LOG.info("ExtraKey in lhs: " + lhsReader.getClass());
      rhsReaderEnd = true;
      break;
    }
  }
  if (!rhsReaderEnd && rhsReader.next()) {
    lhsMissingKeyCounter.increment(1);
    LOG.info("ExtraKey in rhs: " + rhsReader.getClass());
  }
}
Example 9
Source File: GcTimeUpdater.java    From tez with Apache License 2.0 5 votes vote down vote up 
/**
 * Increment the gc-elapsed-time counter.
 */
public void incrementGcCounter() {
  if (null == counters) {
    return; // nothing to do.
  }

  TezCounter gcCounter = counters.findCounter(TaskCounter.GC_TIME_MILLIS);
  if (null != gcCounter) {
    gcCounter.increment(getElapsedGc());
  }
}
Example 10
Source File: IFile.java    From tez with Apache License 2.0 5 votes vote down vote up 
/**
 * Construct an IFile Reader.
 *
 * @param in   The input stream
 * @param length Length of the data in the stream, including the checksum
 *               bytes.
 * @param codec codec
 * @param readsCounter Counter for records read from disk
 * @throws IOException
 */
public Reader(InputStream in, long length,
    CompressionCodec codec,
    TezCounter readsCounter, TezCounter bytesReadCounter,
    boolean readAhead, int readAheadLength,
    int bufferSize) throws IOException {
  this(in, ((in != null) ? (length - HEADER.length) : length), codec,
      readsCounter, bytesReadCounter, readAhead, readAheadLength,
      bufferSize, ((in != null) ? isCompressedFlagEnabled(in) : false));
  if (in != null && bytesReadCounter != null) {
    bytesReadCounter.increment(IFile.HEADER.length);
  }
}
Example 11
Source File: MRInputUtils.java    From tez with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
public static org.apache.hadoop.mapreduce.InputSplit getNewSplitDetailsFromDisk(
    TaskSplitIndex splitMetaInfo, JobConf jobConf, TezCounter splitBytesCounter)
    throws IOException {
  Path file = new Path(splitMetaInfo.getSplitLocation());
  long offset = splitMetaInfo.getStartOffset();

  // Split information read from local filesystem.
  FileSystem fs = FileSystem.getLocal(jobConf);
  file = fs.makeQualified(file);
  LOG.info("Reading input split file from : " + file);
  FSDataInputStream inFile = fs.open(file);
  inFile.seek(offset);
  String className = Text.readString(inFile);
  Class<org.apache.hadoop.mapreduce.InputSplit> cls;
  try {
    cls = (Class<org.apache.hadoop.mapreduce.InputSplit>) jobConf.getClassByName(className);
  } catch (ClassNotFoundException ce) {
    IOException wrap = new IOException("Split class " + className + " not found");
    wrap.initCause(ce);
    throw wrap;
  }
  SerializationFactory factory = new SerializationFactory(jobConf);
  Deserializer<org.apache.hadoop.mapreduce.InputSplit> deserializer = (Deserializer<org.apache.hadoop.mapreduce.InputSplit>) factory
      .getDeserializer(cls);
  deserializer.open(inFile);
  org.apache.hadoop.mapreduce.InputSplit split = deserializer.deserialize(null);
  long pos = inFile.getPos();
  if (splitBytesCounter != null) {
    splitBytesCounter.increment(pos - offset);
  }
  inFile.close();
  return split;
}
Example 12
Source File: MRInputUtils.java    From tez with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
public static InputSplit getOldSplitDetailsFromDisk(TaskSplitIndex splitMetaInfo,
    JobConf jobConf, TezCounter splitBytesCounter) throws IOException {
  Path file = new Path(splitMetaInfo.getSplitLocation());
  FileSystem fs = FileSystem.getLocal(jobConf);
  file = fs.makeQualified(file);
  LOG.info("Reading input split file from : " + file);
  long offset = splitMetaInfo.getStartOffset();

  FSDataInputStream inFile = fs.open(file);
  inFile.seek(offset);
  String className = Text.readString(inFile);
  Class<org.apache.hadoop.mapred.InputSplit> cls;
  try {
    cls = (Class<org.apache.hadoop.mapred.InputSplit>) jobConf.getClassByName(className);
  } catch (ClassNotFoundException ce) {
    IOException wrap = new IOException("Split class " + className + " not found");
    wrap.initCause(ce);
    throw wrap;
  }
  SerializationFactory factory = new SerializationFactory(jobConf);
  Deserializer<org.apache.hadoop.mapred.InputSplit> deserializer = (Deserializer<org.apache.hadoop.mapred.InputSplit>) factory
      .getDeserializer(cls);
  deserializer.open(inFile);
  org.apache.hadoop.mapred.InputSplit split = deserializer.deserialize(null);
  long pos = inFile.getPos();
  if (splitBytesCounter != null) {
    splitBytesCounter.increment(pos - offset);
  }
  inFile.close();
  return split;
}