Java Code Examples for org.apache.tez.common.TezUtils#createConfFromUserPayload()

The following examples show how to use org.apache.tez.common.TezUtils#createConfFromUserPayload() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TezTestServiceContainerLauncher.java    From tez with Apache License 2.0 6 votes vote down vote up
public TezTestServiceContainerLauncher(ContainerLauncherContext containerLauncherContext) {
  super(containerLauncherContext);
  try {
    conf = TezUtils.createConfFromUserPayload(getContext().getInitialUserPayload());
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
  int numThreads = conf.getInt(
      TezTestServiceConfConstants.TEZ_TEST_SERVICE_AM_COMMUNICATOR_NUM_THREADS,
      TezTestServiceConfConstants.TEZ_TEST_SERVICE_AM_COMMUNICATOR_NUM_THREADS_DEFAULT);

  this.servicePort = conf.getInt(
      TezTestServiceConfConstants.TEZ_TEST_SERVICE_RPC_PORT, -1);
  Preconditions.checkArgument(servicePort > 0,
      TezTestServiceConfConstants.TEZ_TEST_SERVICE_RPC_PORT + " must be set");
  this.communicator = new TezTestServiceCommunicator(numThreads);
  this.tokenIdentifier = getContext().getApplicationAttemptId().getApplicationId().toString();
  this.appAttemptId = getContext().getApplicationAttemptId();
}
 
Example 2
Source File: YarnTaskSchedulerService.java    From tez with Apache License 2.0 6 votes vote down vote up
@Private
@VisibleForTesting
YarnTaskSchedulerService(TaskSchedulerContext taskSchedulerContext,
    TezAMRMClientAsync<CookieContainerRequest> client) {
  super(taskSchedulerContext);
  this.containerSignatureMatcher = taskSchedulerContext.getContainerSignatureMatcher();
  this.amRmClient = client;
  this.appHostName = taskSchedulerContext.getAppHostName();
  this.appHostPort = taskSchedulerContext.getAppClientPort();
  this.appTrackingUrl = taskSchedulerContext.getAppTrackingUrl();
  try {
    this.conf = TezUtils.createConfFromUserPayload(taskSchedulerContext.getInitialUserPayload());
  } catch (IOException e) {
    throw new TezUncheckedException(
        "Failed to deserialize payload for " + YarnTaskSchedulerService.class.getSimpleName(),
        e);
  }
}
 
Example 3
Source File: OnFileSortedOutput.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up
@Override
public synchronized List<Event> initialize() throws IOException {
  this.startTime = System.nanoTime();
  this.conf = TezUtils.createConfFromUserPayload(getContext().getUserPayload());
  // Initializing this parametr in this conf since it is used in multiple
  // places (wherever LocalDirAllocator is used) - TezTaskOutputFiles,
  // TezMerger, etc.
  this.conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, getContext().getWorkDirs());
  this.memoryUpdateCallbackHandler = new MemoryUpdateCallbackHandler();
  getContext().requestInitialMemory(
      ExternalSorter.getInitialMemoryRequirement(conf,
          getContext().getTotalMemoryAvailableToTask()), memoryUpdateCallbackHandler);

  sendEmptyPartitionDetails = this.conf.getBoolean(
      TezJobConfig.TEZ_RUNTIME_EMPTY_PARTITION_INFO_VIA_EVENTS_ENABLED,
      TezJobConfig.TEZ_RUNTIME_EMPTY_PARTITION_INFO_VIA_EVENTS_ENABLED_DEFAULT);
  return Collections.emptyList();
}
 
Example 4
Source File: OnFileUnorderedKVOutput.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up
@Override
public synchronized List<Event> initialize()
    throws Exception {
  this.conf = TezUtils.createConfFromUserPayload(getContext()
      .getUserPayload());
  this.conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS,
      getContext().getWorkDirs());

  getContext().requestInitialMemory(0l, null); // mandatory call

  this.dataViaEventsEnabled = conf.getBoolean(
      TezJobConfig.TEZ_RUNTIME_BROADCAST_DATA_VIA_EVENTS_ENABLED,
      TezJobConfig.TEZ_RUNTIME_BROADCAST_DATA_VIA_EVENTS_ENABLED_DEFAULT);
  this.dataViaEventsMaxSize = conf.getInt(
      TezJobConfig.TEZ_RUNTIME_BROADCAST_DATA_VIA_EVENTS_MAX_SIZE,
      TezJobConfig.TEZ_RUNTIME_BROADCAST_DATA_VIA_EVENTS_MAX_SIZE_DEFAULT);
  
  LOG.info(this.getClass().getSimpleName() + " running with params -> "
      + "dataViaEventsEnabled: " + dataViaEventsEnabled
      + ", dataViaEventsMaxSize: " + dataViaEventsMaxSize);
  
  this.kvWriter = new FileBasedKVWriter(getContext(), conf);
  return Collections.emptyList();
}
 
Example 5
Source File: TezContainerLauncherImpl.java    From tez with Apache License 2.0 6 votes vote down vote up
public TezContainerLauncherImpl(ContainerLauncherContext containerLauncherContext) {
  super(containerLauncherContext);
  try {
    this.conf = TezUtils.createConfFromUserPayload(containerLauncherContext.getInitialUserPayload());
  } catch (IOException e) {
    throw new TezUncheckedException(
        "Failed to parse user payload for " + TezContainerLauncherImpl.class.getSimpleName(), e);
  }
  conf.setInt(
      CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY,
      0);
  this.limitOnPoolSize = conf.getInt(
      TezConfiguration.TEZ_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT,
      TezConfiguration.TEZ_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT_DEFAULT);
  LOG.info("Upper limit on the thread pool size is " + this.limitOnPoolSize);
}
 
Example 6
Source File: TestAMRecovery.java    From tez with Apache License 2.0 5 votes vote down vote up
@Override
public void initialize() {
  super.initialize();
  try {
    conf =
        TezUtils.createConfFromUserPayload(getContext().getUserPayload());
  } catch (IOException e) {
    e.printStackTrace();
  }
}
 
Example 7
Source File: MRCombiner.java    From tez with Apache License 2.0 5 votes vote down vote up
public MRCombiner(TaskContext taskContext) throws IOException {
  final Configuration userConf = TezUtils.createConfFromUserPayload(taskContext.getUserPayload());
  useNewApi = ConfigUtils.useNewApi(userConf);
  if (useNewApi) {
    conf = new JobConf(userConf);
  } else {
    conf = userConf;
  }

  assert(taskContext instanceof InputContext || taskContext instanceof OutputContext);
  if (taskContext instanceof OutputContext) {
    this.keyClass = ConfigUtils.getIntermediateOutputKeyClass(conf);
    this.valClass = ConfigUtils.getIntermediateOutputValueClass(conf);
    this.comparator = ConfigUtils.getIntermediateOutputKeyComparator(conf);
    this.reporter = new MRTaskReporter((OutputContext)taskContext);
  } else {
    this.keyClass = ConfigUtils.getIntermediateInputKeyClass(conf);
    this.valClass = ConfigUtils.getIntermediateInputValueClass(conf);
    this.comparator = ConfigUtils.getIntermediateInputKeyComparator(conf);
    this.reporter = new MRTaskReporter((InputContext)taskContext);
  }

  combineInputRecordsCounter = taskContext.getCounters().findCounter(TaskCounter.COMBINE_INPUT_RECORDS);
  combineOutputRecordsCounter = taskContext.getCounters().findCounter(TaskCounter.COMBINE_OUTPUT_RECORDS);
  
  boolean isMap = conf.getBoolean(MRConfig.IS_MAP_PROCESSOR,false);
  this.mrTaskAttemptID = new TaskAttemptID(
      new TaskID(String.valueOf(taskContext.getApplicationId()
          .getClusterTimestamp()), taskContext.getApplicationId().getId(),
          isMap ? TaskType.MAP : TaskType.REDUCE,
          taskContext.getTaskIndex()), taskContext.getTaskAttemptNumber());
  
  LOG.info("Using combineKeyClass: " + keyClass + ", combineValueClass: " + valClass + ", combineComparator: " +comparator + ", useNewApi: " + useNewApi);
}
 
Example 8
Source File: YarnTaskSchedulerService.java    From tez with Apache License 2.0 5 votes vote down vote up
public YarnTaskSchedulerService(TaskSchedulerContext taskSchedulerContext) {
  super(taskSchedulerContext);
  this.containerSignatureMatcher = taskSchedulerContext.getContainerSignatureMatcher();
  this.amRmClient = TezAMRMClientAsync.createAMRMClientAsync(1000, this);
  this.appHostName = taskSchedulerContext.getAppHostName();
  this.appHostPort = taskSchedulerContext.getAppClientPort();
  this.appTrackingUrl = taskSchedulerContext.getAppTrackingUrl();
  try {
    this.conf = TezUtils.createConfFromUserPayload(taskSchedulerContext.getInitialUserPayload());
  } catch (IOException e) {
    throw new TezUncheckedException(
        "Failed to deserialize payload for " + YarnTaskSchedulerService.class.getSimpleName(),
        e);
  }
}
 
Example 9
Source File: MRCombiner.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
public MRCombiner(TezTaskContext taskContext) throws IOException {
  this.conf = TezUtils.createConfFromUserPayload(taskContext.getUserPayload());

  assert(taskContext instanceof TezInputContext || taskContext instanceof TezOutputContext);
  if (taskContext instanceof TezOutputContext) {
    this.keyClass = ConfigUtils.getIntermediateOutputKeyClass(conf);
    this.valClass = ConfigUtils.getIntermediateOutputValueClass(conf);
    this.comparator = ConfigUtils.getIntermediateOutputKeyComparator(conf);
    this.reporter = new MRTaskReporter((TezOutputContext)taskContext);
  } else {
    this.keyClass = ConfigUtils.getIntermediateInputKeyClass(conf);
    this.valClass = ConfigUtils.getIntermediateInputValueClass(conf);
    this.comparator = ConfigUtils.getIntermediateInputKeyComparator(conf);
    this.reporter = new MRTaskReporter((TezInputContext)taskContext);
  }

  this.useNewApi = ConfigUtils.useNewApi(conf);
  
  combineInputKeyCounter = taskContext.getCounters().findCounter(TaskCounter.COMBINE_INPUT_RECORDS);
  combineInputValueCounter = taskContext.getCounters().findCounter(TaskCounter.COMBINE_OUTPUT_RECORDS);
  
  boolean isMap = conf.getBoolean(MRConfig.IS_MAP_PROCESSOR,false);
  this.mrTaskAttemptID = new TaskAttemptID(
      new TaskID(String.valueOf(taskContext.getApplicationId()
          .getClusterTimestamp()), taskContext.getApplicationId().getId(),
          isMap ? TaskType.MAP : TaskType.REDUCE,
          taskContext.getTaskIndex()), taskContext.getTaskAttemptNumber());
  
  LOG.info("Using combineKeyClass: " + keyClass + ", combineValueClass: " + valClass + ", combineComparator: " +comparator + ", useNewApi: " + useNewApi);
}
 
Example 10
Source File: TestEntityDescriptor.java    From tez with Apache License 2.0 5 votes vote down vote up
public void verifyResults(InputDescriptor entityDescriptor, InputDescriptor deserialized, UserPayload payload,
                           String confVal) throws IOException {
  Assert.assertEquals(entityDescriptor.getClassName(), deserialized.getClassName());
  // History text is not serialized when sending to tasks
  Assert.assertNull(deserialized.getHistoryText());
  Assert.assertArrayEquals(payload.deepCopyAsArray(), deserialized.getUserPayload().deepCopyAsArray());
  Configuration deserializedConf = TezUtils.createConfFromUserPayload(deserialized.getUserPayload());
  Assert.assertEquals(confVal, deserializedConf.get("testKey"));
}
 
Example 11
Source File: UnorderedPartitionedKVOutputConfig.java    From tez with Apache License 2.0 5 votes vote down vote up
@InterfaceAudience.Private
public void fromUserPayload(UserPayload payload) {
  try {
    this.conf = TezUtils.createConfFromUserPayload(payload);
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}
 
Example 12
Source File: LocalMergedInput.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
@Override
public List<Event> initialize() throws IOException {
  getContext().requestInitialMemory(0l, null); // mandatory call.
  getContext().inputIsReady();
  this.conf = TezUtils.createConfFromUserPayload(getContext().getUserPayload());

  if (getNumPhysicalInputs() == 0) {
    return Collections.emptyList();
  }

  LocalShuffle localShuffle = new LocalShuffle(getContext(), conf, getNumPhysicalInputs());
  rawIter = localShuffle.run();
  createValuesIterator();
  return Collections.emptyList();
}
 
Example 13
Source File: TestAMRecovery.java    From tez with Apache License 2.0 5 votes vote down vote up
@Override
public void initialize() {
  super.initialize();
  try {
    conf =
        TezUtils.createConfFromUserPayload(getContext().getUserPayload());
  } catch (IOException e) {
    e.printStackTrace();
  }
}
 
Example 14
Source File: TestAMRecovery.java    From tez with Apache License 2.0 5 votes vote down vote up
@Override
public void initialize() {
  super.initialize();
  try {
    conf =
        TezUtils.createConfFromUserPayload(getContext().getUserPayload());
  } catch (IOException e) {
    e.printStackTrace();
  }
}
 
Example 15
Source File: OnFileUnorderedPartitionedKVOutput.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
@Override
public synchronized List<Event> initialize(TezOutputContext outputContext) throws Exception {
  this.outputContext = outputContext;
  this.conf = TezUtils.createConfFromUserPayload(outputContext.getUserPayload());
  this.conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, outputContext.getWorkDirs());
  this.conf.setInt(TezRuntimeFrameworkConfigs.TEZ_RUNTIME_NUM_EXPECTED_PARTITIONS,
      this.numPhysicalOutputs);
  this.memoryUpdateCallbackHandler = new MemoryUpdateCallbackHandler();
  outputContext.requestInitialMemory(
      UnorderedPartitionedKVWriter.getInitialMemoryRequirement(conf,
          outputContext.getTotalMemoryAvailableToTask()), memoryUpdateCallbackHandler);
  return Collections.emptyList();
}
 
Example 16
Source File: TezDAGStats.java    From spork with Apache License 2.0 4 votes vote down vote up
/**
 * Updates the statistics after a DAG is finished.
 */
public void accumulateStats(TezJob tezJob) throws IOException {
    //For Oozie Pig action job id matching compatibility with MR mode
    appId = tezJob.getApplicationId().toString().replace("application", "job");
    setConf(tezJob.getConfiguration());
    DAG dag = tezJob.getDAG();
    hdfsBytesRead = -1;
    hdfsBytesWritten = -1;
    TezCounters tezCounters = tezJob.getDAGCounters();
    if (tezCounters != null) {
        counters = covertToHadoopCounters(tezCounters);

        CounterGroup dagGrp = tezCounters.getGroup(DAG_COUNTER_GROUP);
        totalTasks = (int) dagGrp.findCounter("TOTAL_LAUNCHED_TASKS").getValue();

        CounterGroup fsGrp = tezCounters.getGroup(FS_COUNTER_GROUP);
        fileBytesRead = fsGrp.findCounter("FILE_BYTES_READ").getValue();
        fileBytesWritten = fsGrp.findCounter("FILE_BYTES_WRITTEN").getValue();
        hdfsBytesRead = fsGrp.findCounter("HDFS_BYTES_READ").getValue();
        hdfsBytesWritten = fsGrp.findCounter("HDFS_BYTES_WRITTEN").getValue();
    } else {
        LOG.warn("Failed to get counters for DAG: " + dag.getName());
    }

    for (Entry<String, TezVertexStats> entry : tezVertexStatsMap.entrySet()) {
        Vertex v = dag.getVertex(entry.getKey());
        if (v != null && tezVertexStatsMap.containsKey(v.getName())) {
            TezVertexStats vertexStats = entry.getValue();
            UserPayload payload = v.getProcessorDescriptor().getUserPayload();
            Configuration conf = TezUtils.createConfFromUserPayload(payload);
            vertexStats.setConf(conf);

            VertexStatus status = tezJob.getVertexStatus(v.getName());
            if (status == null) {
                LOG.warn("Failed to get status for vertex: " + v.getName());
                continue;
            }
            vertexStats.accumulateStats(status, v.getParallelism());
            if(vertexStats.getInputs() != null && !vertexStats.getInputs().isEmpty()) {
                inputs.addAll(vertexStats.getInputs());
            }
            if(vertexStats.getOutputs() != null  && !vertexStats.getOutputs().isEmpty()) {
                outputs.addAll(vertexStats.getOutputs());
            }
            /*if (vertexStats.getHdfsBytesRead() >= 0) {
                hdfsBytesRead = (hdfsBytesRead == -1) ? 0 : hdfsBytesRead;
                hdfsBytesRead += vertexStats.getHdfsBytesRead();
            }
            if (vertexStats.getHdfsBytesWritten() >= 0) {
                hdfsBytesWritten = (hdfsBytesWritten == -1) ? 0 : hdfsBytesWritten;
                hdfsBytesWritten += vertexStats.getHdfsBytesWritten();
            }*/
            if (!vertexStats.getMultiStoreCounters().isEmpty()) {
                multiStoreCounters.putAll(vertexStats.getMultiStoreCounters());
            }
            numberMaps += vertexStats.getNumberMaps();
            numberReduces += vertexStats.getNumberReduces();
            mapInputRecords += vertexStats.getMapInputRecords();
            mapOutputRecords += vertexStats.getMapOutputRecords();
            reduceInputRecords += vertexStats.getReduceInputRecords();
            reduceOutputRecords += vertexStats.getReduceOutputRecords();
            spillCount += vertexStats.getSMMSpillCount();
            activeSpillCountObj  += vertexStats.getProactiveSpillCountObjects();
            activeSpillCountRecs += vertexStats.getProactiveSpillCountRecs();
        }
    }
}
 
Example 17
Source File: RootInputVertexManager.java    From tez with Apache License 2.0 4 votes vote down vote up
@Override
public void initialize() {
  UserPayload userPayload = getContext().getUserPayload();
  if (userPayload == null || userPayload.getPayload() == null ||
      userPayload.getPayload().limit() == 0) {
    throw new RuntimeException("Could not initialize RootInputVertexManager"
        + " from provided user payload");
  }
  try {
    conf = TezUtils.createConfFromUserPayload(userPayload);
  } catch (IOException e) {
    throw new RuntimeException("Could not initialize RootInputVertexManager"
      + " from provided user payload", e);
  }
  slowStartEnabled = conf.getBoolean(
    TEZ_ROOT_INPUT_VERTEX_MANAGER_ENABLE_SLOW_START,
    TEZ_ROOT_INPUT_VERTEX_MANAGER_ENABLE_SLOW_START_DEFAULT);

  if(slowStartEnabled) {
    slowStartMinFraction = conf.getFloat(
      TEZ_ROOT_INPUT_VERTEX_MANAGER_MIN_SRC_FRACTION,
      TEZ_ROOT_INPUT_VERTEX_MANAGER_MIN_SRC_FRACTION_DEFAULT);
    slowStartMaxFraction = conf.getFloat(
      TEZ_ROOT_INPUT_VERTEX_MANAGER_MAX_SRC_FRACTION,
      Math.max(slowStartMinFraction,
          TEZ_ROOT_INPUT_VERTEX_MANAGER_MAX_SRC_FRACTION_DEFAULT));
  } else {
    slowStartMinFraction = 0;
    slowStartMaxFraction = 0;
  }
  if (slowStartMinFraction < 0 || slowStartMaxFraction > 1
      || slowStartMaxFraction < slowStartMinFraction) {
    throw new IllegalArgumentException(
        "Invalid values for slowStartMinFraction"
        + "/slowStartMaxFraction. Min "
        + "cannot be < 0, max cannot be > 1, and max cannot be < min."
        + ", configuredMin=" + slowStartMinFraction
        + ", configuredMax=" + slowStartMaxFraction);
  }


  updatePendingTasks();
}
 
Example 18
Source File: MRHelpers.java    From incubator-tez with Apache License 2.0 4 votes vote down vote up
@LimitedPrivate("Hive, Pig")
@Unstable
public static Configuration createConfFromUserPayload(byte[] bb)
    throws IOException {
  return TezUtils.createConfFromUserPayload(bb);
}
 
Example 19
Source File: TestTaskSchedulerManager.java    From tez with Apache License 2.0 4 votes vote down vote up
@Test(timeout = 5000)
public void testCustomTaskSchedulerSetup() throws IOException {
  Configuration conf = new Configuration(false);
  conf.set("testkey", "testval");
  UserPayload defaultPayload = TezUtils.createUserPayloadFromConf(conf);

  String customSchedulerName = "fakeScheduler";
  List<NamedEntityDescriptor> taskSchedulers = new LinkedList<>();
  ByteBuffer bb = ByteBuffer.allocate(4);
  bb.putInt(0, 3);
  UserPayload userPayload = UserPayload.create(bb);
  taskSchedulers.add(
      new NamedEntityDescriptor(customSchedulerName, FakeTaskScheduler.class.getName())
          .setUserPayload(userPayload));
  taskSchedulers.add(new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null)
      .setUserPayload(defaultPayload));

  TSEHForMultipleSchedulersTest tseh =
      new TSEHForMultipleSchedulersTest(mockAppContext, mockClientService, mockEventHandler,
          mockSigMatcher, mockWebUIService, taskSchedulers, false);

  tseh.init(conf);
  tseh.start();

  // Verify that the YARN task scheduler is installed by default
  assertTrue(tseh.getYarnSchedulerCreated());
  assertFalse(tseh.getUberSchedulerCreated());
  assertEquals(2, tseh.getNumCreateInvocations());

  // Verify the order of the schedulers
  assertEquals(customSchedulerName, tseh.getTaskSchedulerName(0));
  assertEquals(TezConstants.getTezYarnServicePluginName(), tseh.getTaskSchedulerName(1));

  // Verify the payload setup for the custom task scheduler
  assertNotNull(tseh.getTaskSchedulerContext(0));
  assertEquals(bb, tseh.getTaskSchedulerContext(0).getInitialUserPayload().getPayload());

  // Verify the payload on the yarn scheduler
  assertNotNull(tseh.getTaskSchedulerContext(1));
  Configuration parsed = TezUtils.createConfFromUserPayload(tseh.getTaskSchedulerContext(1).getInitialUserPayload());
  assertEquals("testval", parsed.get("testkey"));
}
 
Example 20
Source File: PigProcessor.java    From spork with Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public void initialize() throws Exception {
    // Reset any static variables to avoid conflict in container-reuse.
    sampleVertex = null;
    sampleMap = null;

    // Reset static variables cleared for avoiding OOM.
    new JVMReuseImpl().cleanupStaticData();

    UserPayload payload = getContext().getUserPayload();
    conf = TezUtils.createConfFromUserPayload(payload);
    PigContext.setPackageImportList((ArrayList<String>) ObjectSerializer
            .deserialize(conf.get("udf.import.list")));
    PigContext pc = (PigContext) ObjectSerializer.deserialize(conf.get("pig.pigContext"));

    // To determine front-end in UDFContext
    conf.set(MRConfiguration.JOB_APPLICATION_ATTEMPT_ID, getContext().getUniqueIdentifier());
    conf.set(PigConstants.TASK_INDEX, Integer.toString(getContext().getTaskIndex()));
    UDFContext.getUDFContext().addJobConf(conf);
    UDFContext.getUDFContext().deserialize();

    String execPlanString = conf.get(PLAN);
    execPlan = (PhysicalPlan) ObjectSerializer.deserialize(execPlanString);
    SchemaTupleBackend.initialize(conf, pc);
    PigMapReduce.sJobContext = HadoopShims.createJobContext(conf, new org.apache.hadoop.mapreduce.JobID());

    // Set the job conf as a thread-local member of PigMapReduce
    // for backwards compatibility with the existing code base.
    PigMapReduce.sJobConfInternal.set(conf);

    Utils.setDefaultTimeZone(conf);

    boolean aggregateWarning = "true".equalsIgnoreCase(pc.getProperties().getProperty("aggregate.warning"));
    PigStatusReporter pigStatusReporter = PigStatusReporter.getInstance();
    pigStatusReporter.setContext(new TezTaskContext(getContext()));
    pigHadoopLogger = PigHadoopLogger.getInstance();
    pigHadoopLogger.setReporter(pigStatusReporter);
    pigHadoopLogger.setAggregate(aggregateWarning);
    PhysicalOperator.setPigLogger(pigHadoopLogger);

    LinkedList<TezTaskConfigurable> tezTCs = PlanHelper.getPhysicalOperators(execPlan, TezTaskConfigurable.class);
    for (TezTaskConfigurable tezTC : tezTCs){
        tezTC.initialize(getContext());
    }
}