Java Code Examples for org.apache.tez.runtime.library.common.ConfigUtils#getInputKeySecondaryGroupingComparator()

The following examples show how to use org.apache.tez.runtime.library.common.ConfigUtils#getInputKeySecondaryGroupingComparator() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: POShuffleTezLoad.java From spork with Apache License 2.0

5 votes

@Override
public void attachInputs(Map<String, LogicalInput> inputs, Configuration conf)
        throws ExecException {
    this.conf = conf;
    comparator = (WritableComparator) ConfigUtils.getInputKeySecondaryGroupingComparator(conf);
    try {
        for (String key : inputKeys) {
            LogicalInput input = inputs.get(key);
            this.inputs.add(input);
            this.readers.add((KeyValuesReader)input.getReader());
        }

        // We need to adjust numInputs because it's possible for both
        // OrderedGroupedKVInput and non-OrderedGroupedKVInput to be attached
        // to the same vertex. If so, we're only interested in
        // OrderedGroupedKVInputs. So we ignore the others.
        this.numInputs = this.inputs.size();

        readOnce = new boolean[numInputs];
        for (int i = 0; i < numInputs; i++) {
            readOnce[i] = false;
        }

        finished = new boolean[numInputs];
        for (int i = 0; i < numInputs; i++) {
            finished[i] = !readers.get(i).next();
        }
    } catch (Exception e) {
        throw new ExecException(e);
    }
    accumulativeBatchSize = AccumulatorOptimizerUtil.getAccumulativeBatchSize();
}

Example 2

Source File: ReduceProcessor.java From incubator-tez with Apache License 2.0

4 votes

@Override
public void run(Map<String, LogicalInput> inputs,
    Map<String, LogicalOutput> outputs) throws Exception {

  LOG.info("Running reduce: " + processorContext.getUniqueIdentifier());

  if (outputs.size() <= 0 || outputs.size() > 1) {
    throw new IOException("Invalid number of outputs"
        + ", outputCount=" + outputs.size());
  }

  if (inputs.size() <= 0 || inputs.size() > 1) {
    throw new IOException("Invalid number of inputs"
        + ", inputCount=" + inputs.size());
  }

  LogicalInput in = inputs.values().iterator().next();
  in.start();

  List<Input> pendingInputs = new LinkedList<Input>();
  pendingInputs.add(in);
  processorContext.waitForAllInputsReady(pendingInputs);
  LOG.info("Input is ready for consumption. Starting Output");

  LogicalOutput out = outputs.values().iterator().next();
  out.start();

  initTask(out);

  this.statusUpdate();

  Class keyClass = ConfigUtils.getIntermediateInputKeyClass(jobConf);
  Class valueClass = ConfigUtils.getIntermediateInputValueClass(jobConf);
  LOG.info("Using keyClass: " + keyClass);
  LOG.info("Using valueClass: " + valueClass);
  RawComparator comparator =
      ConfigUtils.getInputKeySecondaryGroupingComparator(jobConf);
  LOG.info("Using comparator: " + comparator);

  reduceInputKeyCounter =
      mrReporter.getCounter(TaskCounter.REDUCE_INPUT_GROUPS);
  reduceInputValueCounter =
      mrReporter.getCounter(TaskCounter.REDUCE_INPUT_RECORDS);

  // Sanity check
  if (!(in instanceof ShuffledMergedInputLegacy)) {
    throw new IOException("Illegal input to reduce: " + in.getClass());
  }
  ShuffledMergedInputLegacy shuffleInput = (ShuffledMergedInputLegacy)in;
  KeyValuesReader kvReader = shuffleInput.getReader();

  KeyValueWriter kvWriter = null;
  if((out instanceof MROutputLegacy)) {
    kvWriter = ((MROutputLegacy) out).getWriter();
  } else if ((out instanceof OnFileSortedOutput)) {
    kvWriter = ((OnFileSortedOutput) out).getWriter();
  } else {
    throw new IOException("Illegal output to reduce: " + in.getClass());
  }

  if (useNewApi) {
    try {
      runNewReducer(
          jobConf,
          mrReporter,
          shuffleInput, comparator,  keyClass, valueClass,
          kvWriter);
    } catch (ClassNotFoundException cnfe) {
      throw new IOException(cnfe);
    }
  } else {
    runOldReducer(
        jobConf, mrReporter,
        kvReader, comparator, keyClass, valueClass, kvWriter);
  }

  done();
}

Example 3

Source File: ReduceProcessor.java From tez with Apache License 2.0

4 votes

@Override
public void run(Map<String, LogicalInput> _inputs,
    Map<String, LogicalOutput> _outputs) throws Exception {
  this.inputs = _inputs;
  this.outputs = _outputs;
  progressHelper = new ProgressHelper(this.inputs, processorContext, this.getClass().getSimpleName());
  LOG.info("Running reduce: " + processorContext.getUniqueIdentifier());

  if (_outputs.size() <= 0 || _outputs.size() > 1) {
    throw new IOException("Invalid number of _outputs"
        + ", outputCount=" + _outputs.size());
  }

  if (_inputs.size() <= 0 || _inputs.size() > 1) {
    throw new IOException("Invalid number of _inputs"
        + ", inputCount=" + _inputs.size());
  }

  LogicalInput in = _inputs.values().iterator().next();
  in.start();

  List<Input> pendingInputs = new LinkedList<Input>();
  pendingInputs.add(in);
  processorContext.waitForAllInputsReady(pendingInputs);
  LOG.info("Input is ready for consumption. Starting Output");

  LogicalOutput out = _outputs.values().iterator().next();
  out.start();

  initTask(out);
  progressHelper.scheduleProgressTaskService(0, 100);
  this.statusUpdate();

  Class keyClass = ConfigUtils.getIntermediateInputKeyClass(jobConf);
  Class valueClass = ConfigUtils.getIntermediateInputValueClass(jobConf);
  LOG.info("Using keyClass: " + keyClass);
  LOG.info("Using valueClass: " + valueClass);
  RawComparator comparator =
      ConfigUtils.getInputKeySecondaryGroupingComparator(jobConf);
  LOG.info("Using comparator: " + comparator);

  reduceInputKeyCounter =
      mrReporter.getCounter(TaskCounter.REDUCE_INPUT_GROUPS);
  reduceInputValueCounter =
      mrReporter.getCounter(TaskCounter.REDUCE_INPUT_RECORDS);

  // Sanity check
  if (!(in instanceof OrderedGroupedInputLegacy)) {
    throw new IOException("Illegal input to reduce: " + in.getClass());
  }
  OrderedGroupedInputLegacy shuffleInput = (OrderedGroupedInputLegacy)in;
  KeyValuesReader kvReader = shuffleInput.getReader();

  KeyValueWriter kvWriter = null;
  if((out instanceof MROutputLegacy)) {
    kvWriter = ((MROutputLegacy) out).getWriter();
  } else if ((out instanceof OrderedPartitionedKVOutput)) {
    kvWriter = ((OrderedPartitionedKVOutput) out).getWriter();
  } else {
    throw new IOException("Illegal output to reduce: " + in.getClass());
  }

  if (useNewApi) {
    try {
      runNewReducer(
          jobConf,
          mrReporter,
          shuffleInput, comparator,  keyClass, valueClass,
          kvWriter);
    } catch (ClassNotFoundException cnfe) {
      throw new IOException(cnfe);
    }
  } else {
    runOldReducer(
        jobConf, mrReporter,
        kvReader, comparator, keyClass, valueClass, kvWriter);
  }

  done();
}