Java Code Examples for org.apache.tez.mapreduce.hadoop.DeprecatedKeys#getMRToDAGParamMap()

The following examples show how to use org.apache.tez.mapreduce.hadoop.DeprecatedKeys#getMRToDAGParamMap() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: YARNRunner.java From incubator-tez with Apache License 2.0

6 votes

private TezConfiguration getDAGAMConfFromMRConf() {
  TezConfiguration finalConf = new TezConfiguration(this.tezConf);
  Map<String, String> mrParamToDAGParamMap = DeprecatedKeys
      .getMRToDAGParamMap();

  for (Entry<String, String> entry : mrParamToDAGParamMap.entrySet()) {
    if (finalConf.get(entry.getKey()) != null) {
      finalConf.set(entry.getValue(), finalConf.get(entry.getKey()));
      finalConf.unset(entry.getKey());
      if (LOG.isDebugEnabled()) {
        LOG.debug("MR->DAG Translating MR key: " + entry.getKey()
            + " to Tez key: " + entry.getValue() + " with value "
            + finalConf.get(entry.getValue()));
      }
    }
  }
  return finalConf;
}

Example 2

Source File: YARNRunner.java From tez with Apache License 2.0

6 votes

private TezConfiguration getDAGAMConfFromMRConf() {
  TezConfiguration finalConf = new TezConfiguration(this.tezConf);
  Map<String, String> mrParamToDAGParamMap = DeprecatedKeys
      .getMRToDAGParamMap();

  for (Entry<String, String> entry : mrParamToDAGParamMap.entrySet()) {
    if (finalConf.get(entry.getKey()) != null) {
      finalConf.set(entry.getValue(), finalConf.get(entry.getKey()));
      finalConf.unset(entry.getKey());
      if (LOG.isDebugEnabled()) {
        LOG.debug("MR->DAG Translating MR key: " + entry.getKey()
            + " to Tez key: " + entry.getValue() + " with value "
            + finalConf.get(entry.getValue()));
      }
    }
  }
  return finalConf;
}

Example 3

Source File: MRToTezHelper.java From spork with Apache License 2.0

4 votes

public static TezConfiguration getDAGAMConfFromMRConf(
        Configuration tezConf) {

    // Set Tez parameters based on MR parameters.
    TezConfiguration dagAMConf = new TezConfiguration(tezConf);
    Map<String, String> mrParamToDAGParamMap = DeprecatedKeys
            .getMRToDAGParamMap();

    for (Entry<String, String> entry : mrParamToDAGParamMap.entrySet()) {
        if (dagAMConf.get(entry.getKey()) != null) {
            dagAMConf.set(entry.getValue(), dagAMConf.get(entry.getKey()));
            dagAMConf.unset(entry.getKey());
            if (LOG.isDebugEnabled()) {
                LOG.debug("MR->DAG Translating MR key: " + entry.getKey()
                        + " to Tez key: " + entry.getValue()
                        + " with value " + dagAMConf.get(entry.getValue()));
            }
        }
    }

    String env = tezConf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV);
    if (tezConf.get(MRJobConfig.MR_AM_ENV) != null) {
        env = (env == null) ? tezConf.get(MRJobConfig.MR_AM_ENV)
                            : env + "," + tezConf.get(MRJobConfig.MR_AM_ENV);
    }

    if (env != null) {
        dagAMConf.setIfUnset(TezConfiguration.TEZ_AM_LAUNCH_ENV, env);
    }

    dagAMConf.setIfUnset(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS,
            org.apache.tez.mapreduce.hadoop.MRHelpers
                    .getJavaOptsForMRAM(tezConf));

    String queueName = tezConf.get(JobContext.QUEUE_NAME,
            YarnConfiguration.DEFAULT_QUEUE_NAME);
    dagAMConf.setIfUnset(TezConfiguration.TEZ_QUEUE_NAME, queueName);

    int amMemMB = tezConf.getInt(MRJobConfig.MR_AM_VMEM_MB,
            MRJobConfig.DEFAULT_MR_AM_VMEM_MB);
    int amCores = tezConf.getInt(MRJobConfig.MR_AM_CPU_VCORES,
            MRJobConfig.DEFAULT_MR_AM_CPU_VCORES);
    dagAMConf.setIfUnset(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, ""
            + amMemMB);
    dagAMConf.setIfUnset(TezConfiguration.TEZ_AM_RESOURCE_CPU_VCORES, ""
            + amCores);

    dagAMConf.setIfUnset(TezConfiguration.TEZ_AM_VIEW_ACLS,
            tezConf.get(MRJobConfig.JOB_ACL_VIEW_JOB, MRJobConfig.DEFAULT_JOB_ACL_VIEW_JOB));

    dagAMConf.setIfUnset(TezConfiguration.TEZ_AM_MODIFY_ACLS,
            tezConf.get(MRJobConfig.JOB_ACL_MODIFY_JOB, MRJobConfig.DEFAULT_JOB_ACL_MODIFY_JOB));

    dagAMConf.setIfUnset(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS, ""
            + dagAMConf.getInt(MRJobConfig.MR_AM_MAX_ATTEMPTS,
                    MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS));

    if (tezConf.get(MRConfiguration.JOB_CREDENTIALS_BINARY) != null) {
        dagAMConf.setIfUnset(TezConfiguration.TEZ_CREDENTIALS_PATH,
                tezConf.get(MRConfiguration.JOB_CREDENTIALS_BINARY));
    }

    //TODO: Strip out all MR settings

    return dagAMConf;
}