Java Code Examples for org.apache.tez.common.TezUtils#createConfFromBaseConfAndPayload()

The following examples show how to use org.apache.tez.common.TezUtils#createConfFromBaseConfAndPayload() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: UnorderedKVOutput.java From tez with Apache License 2.0

6 votes

@Override
public synchronized List<Event> initialize()
    throws Exception {
  this.conf = TezUtils.createConfFromBaseConfAndPayload(getContext());
  this.conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS,
      getContext().getWorkDirs());

  this.conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_PARTITIONER_CLASS, CustomPartitioner.class
      .getName());

  this.memoryUpdateCallbackHandler = new MemoryUpdateCallbackHandler();

  boolean pipelinedShuffle = this.conf.getBoolean(TezRuntimeConfiguration
      .TEZ_RUNTIME_PIPELINED_SHUFFLE_ENABLED, TezRuntimeConfiguration
      .TEZ_RUNTIME_PIPELINED_SHUFFLE_ENABLED_DEFAULT);

  long memRequestSize = (pipelinedShuffle) ?
      UnorderedPartitionedKVWriter.getInitialMemoryRequirement(conf, getContext()
          .getTotalMemoryAvailableToTask()) : 0;
  getContext().requestInitialMemory(memRequestSize, memoryUpdateCallbackHandler);
  
  return Collections.emptyList();
}

Example 2

Source File: OrderedPartitionedKVOutput.java From tez with Apache License 2.0

6 votes

@Override
public synchronized List<Event> initialize() throws IOException {
  this.startTime = System.nanoTime();
  this.conf = TezUtils.createConfFromBaseConfAndPayload(getContext());
  this.localFs = (RawLocalFileSystem) FileSystem.getLocal(conf).getRaw();

  // Initializing this parametr in this conf since it is used in multiple
  // places (wherever LocalDirAllocator is used) - TezTaskOutputFiles,
  // TezMerger, etc.
  this.conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, getContext().getWorkDirs());
  this.memoryUpdateCallbackHandler = new MemoryUpdateCallbackHandler();
  getContext().requestInitialMemory(
      ExternalSorter.getInitialMemoryRequirement(conf,
          getContext().getTotalMemoryAvailableToTask()), memoryUpdateCallbackHandler);

  sendEmptyPartitionDetails = conf.getBoolean(
      TezRuntimeConfiguration.TEZ_RUNTIME_EMPTY_PARTITION_INFO_VIA_EVENTS_ENABLED,
      TezRuntimeConfiguration.TEZ_RUNTIME_EMPTY_PARTITION_INFO_VIA_EVENTS_ENABLED_DEFAULT);

  return Collections.emptyList();
}

Example 3

Source File: UnorderedKVInput.java From tez with Apache License 2.0

6 votes

@Override
public synchronized List<Event> initialize() throws Exception {
  Preconditions.checkArgument(getNumPhysicalInputs() != -1, "Number of Inputs has not been set");
  this.conf = TezUtils.createConfFromBaseConfAndPayload(getContext());

  if (getNumPhysicalInputs() == 0) {
    getContext().requestInitialMemory(0l, null);
    isStarted.set(true);
    getContext().inputIsReady();
    LOG.info("input fetch not required since there are 0 physical inputs for input vertex: "
        + getContext().getSourceVertexName());
    return Collections.emptyList();
  } else {
    long initalMemReq = getInitialMemoryReq();
    memoryUpdateCallbackHandler = new MemoryUpdateCallbackHandler();
    this.getContext().requestInitialMemory(initalMemReq, memoryUpdateCallbackHandler);
  }

  this.conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, getContext().getWorkDirs());
  this.inputRecordCounter = getContext().getCounters().findCounter(
      TaskCounter.INPUT_RECORDS_PROCESSED);
  return Collections.emptyList();
}

Example 4

Source File: UnorderedPartitionedKVOutput.java From tez with Apache License 2.0

5 votes

@Override
public synchronized List<Event> initialize() throws Exception {
  this.conf = TezUtils.createConfFromBaseConfAndPayload(getContext());
  this.conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, getContext().getWorkDirs());
  this.conf.setInt(TezRuntimeFrameworkConfigs.TEZ_RUNTIME_NUM_EXPECTED_PARTITIONS,
      getNumPhysicalOutputs());
  this.memoryUpdateCallbackHandler = new MemoryUpdateCallbackHandler();
  getContext().requestInitialMemory(
      UnorderedPartitionedKVWriter.getInitialMemoryRequirement(conf,
          getContext().getTotalMemoryAvailableToTask()), memoryUpdateCallbackHandler);
  return Collections.emptyList();
}

Example 5

Source File: OrderedGroupedKVInput.java From tez with Apache License 2.0

5 votes

@Override
public synchronized List<Event> initialize() throws IOException {
  this.conf = TezUtils.createConfFromBaseConfAndPayload(getContext());

  if (this.getNumPhysicalInputs() == 0) {
    getContext().requestInitialMemory(0l, null);
    isStarted.set(true);
    getContext().inputIsReady();
    LOG.info("input fetch not required since there are 0 physical inputs for input vertex: "
        + getContext().getSourceVertexName());
    return Collections.emptyList();
  }

  long initialMemoryRequest = Shuffle.getInitialMemoryRequirement(conf,
      getContext().getTotalMemoryAvailableToTask());
  this.memoryUpdateCallbackHandler = new MemoryUpdateCallbackHandler();
  getContext().requestInitialMemory(initialMemoryRequest, memoryUpdateCallbackHandler);

  this.inputKeyCounter = getContext().getCounters().findCounter(TaskCounter.REDUCE_INPUT_GROUPS);
  this.inputValueCounter = getContext().getCounters().findCounter(
      TaskCounter.REDUCE_INPUT_RECORDS);
   this.shuffledInputs = getContext().getCounters().findCounter(
      TaskCounter.NUM_SHUFFLED_INPUTS);
  this.conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, getContext().getWorkDirs());

  return Collections.emptyList();
}