Java Code Examples for org.apache.tez.dag.api.TezConfiguration#get()

The following examples show how to use org.apache.tez.dag.api.TezConfiguration#get() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: YARNRunner.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up 
private TezConfiguration getDAGAMConfFromMRConf() {
  TezConfiguration finalConf = new TezConfiguration(this.tezConf);
  Map<String, String> mrParamToDAGParamMap = DeprecatedKeys
      .getMRToDAGParamMap();

  for (Entry<String, String> entry : mrParamToDAGParamMap.entrySet()) {
    if (finalConf.get(entry.getKey()) != null) {
      finalConf.set(entry.getValue(), finalConf.get(entry.getKey()));
      finalConf.unset(entry.getKey());
      if (LOG.isDebugEnabled()) {
        LOG.debug("MR->DAG Translating MR key: " + entry.getKey()
            + " to Tez key: " + entry.getValue() + " with value "
            + finalConf.get(entry.getValue()));
      }
    }
  }
  return finalConf;
}
Example 2
Source File: YARNRunner.java    From tez with Apache License 2.0 6 votes vote down vote up 
private TezConfiguration getDAGAMConfFromMRConf() {
  TezConfiguration finalConf = new TezConfiguration(this.tezConf);
  Map<String, String> mrParamToDAGParamMap = DeprecatedKeys
      .getMRToDAGParamMap();

  for (Entry<String, String> entry : mrParamToDAGParamMap.entrySet()) {
    if (finalConf.get(entry.getKey()) != null) {
      finalConf.set(entry.getValue(), finalConf.get(entry.getKey()));
      finalConf.unset(entry.getKey());
      if (LOG.isDebugEnabled()) {
        LOG.debug("MR->DAG Translating MR key: " + entry.getKey()
            + " to Tez key: " + entry.getValue() + " with value "
            + finalConf.get(entry.getValue()));
      }
    }
  }
  return finalConf;
}
Example 3
Source File: TezClientUtils.java    From tez with Apache License 2.0 6 votes vote down vote up 
@Private
@VisibleForTesting
static String constructAMLaunchOpts(TezConfiguration tezConf, Resource capability) {
  String defaultOpts = tezConf.get(TezConfiguration.TEZ_AM_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS,
      TezConfiguration.TEZ_AM_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS_DEFAULT);
  Path tmpDir = new Path(Environment.PWD.$(),
      YarnConfiguration.DEFAULT_CONTAINER_TEMP_DIR);
  String amOpts = "-Djava.io.tmpdir=" + tmpDir + " ";

  if (defaultOpts != null && !defaultOpts.isEmpty()) {
    amOpts = amOpts + defaultOpts + " ";
  }
  amOpts = amOpts + tezConf.get(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS,
      TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS_DEFAULT);

  amOpts = maybeAddDefaultMemoryJavaOpts(amOpts, capability,
      tezConf.getDouble(TezConfiguration.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION,
          TezConfiguration.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_DEFAULT));

  return amOpts;
}
Example 4
Source File: TezClientUtils.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up 
static void setDefaultLaunchCmdOpts(Vertex v, TezConfiguration conf) {
  String vOpts = v.getTaskLaunchCmdOpts();
  String vConfigOpts = conf.get(TezConfiguration.TEZ_TASK_LAUNCH_CMD_OPTS,
      TezConfiguration.TEZ_TASK_LAUNCH_CMD_OPTS_DEFAULT);
  if (vConfigOpts != null && vConfigOpts.length() > 0) {
    vOpts += (" " + vConfigOpts);
  }
  
  vOpts = maybeAddDefaultLoggingJavaOpts(conf.get(
      TezConfiguration.TEZ_TASK_LOG_LEVEL,
      TezConfiguration.TEZ_TASK_LOG_LEVEL_DEFAULT), vOpts);
  v.setTaskLaunchCmdOpts(vOpts);
}
Example 5
Source File: TezClientUtils.java    From tez with Apache License 2.0 5 votes vote down vote up 
static Credentials prepareAmLaunchCredentials(AMConfiguration amConfig, Credentials sessionCreds,
    TezConfiguration conf, Path binaryConfPath) throws IOException {
  // Setup security tokens
  Credentials amLaunchCredentials = new Credentials();

  // Add SimpleHistoryLoggingService logDir creds to the list of session credentials
  // If it is on HDFS
  String simpleHistoryLogDir = conf.get(TezConfiguration.TEZ_SIMPLE_HISTORY_LOGGING_DIR);
  if (simpleHistoryLogDir != null && !simpleHistoryLogDir.isEmpty()) {
    Path simpleHistoryLogDirPath = new Path(simpleHistoryLogDir);
    TokenCache.obtainTokensForFileSystems(sessionCreds, new Path[] { simpleHistoryLogDirPath },
        conf);
  }

  // Add Staging dir creds to the list of session credentials.
  TokenCache.obtainTokensForFileSystems(sessionCreds, new Path[] {binaryConfPath }, conf);

  populateTokenCache(conf, sessionCreds);

  // Add session specific credentials to the AM credentials.
  amLaunchCredentials.mergeAll(sessionCreds);

  if (amConfig.getCredentials() != null) {
    amLaunchCredentials.mergeAll(amConfig.getCredentials());
  }
  TezCommonUtils.logCredentials(LOG, amLaunchCredentials, "amLaunch");
  return amLaunchCredentials;
}
Example 6
Source File: DAGClientTimelineImpl.java    From tez with Apache License 2.0 5 votes vote down vote up 
public DAGClientTimelineImpl(ApplicationId appId, String dagId, TezConfiguration conf,
                             FrameworkClient frameworkClient, int connTimeout)
    throws TezException {

  if (!TimelineReaderFactory.isTimelineClientSupported()) {
    throw new TezException("Reading from secure timeline is supported only for hadoop 2.6 and above.");
  }

  this.appId = appId;
  this.dagId = dagId;
  this.frameworkClient = frameworkClient;

  String scheme;
  String webAppAddress;
  boolean useHttps = webappHttpsOnly(conf);
  if (useHttps) {
    scheme = HTTPS_SCHEME;
    webAppAddress = conf.get(ATSConstants.TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS_CONF_NAME);
  } else {
    scheme = HTTP_SCHEME;
    webAppAddress = conf.get(ATSConstants.TIMELINE_SERVICE_WEBAPP_HTTP_ADDRESS_CONF_NAME);
  }
  if (webAppAddress == null) {
    throw new TezException("Failed to get ATS webapp address");
  }

  baseUri = Joiner.on("").join(scheme, webAppAddress, ATSConstants.RESOURCE_URI_BASE);

  timelineReaderStrategy =
      TimelineReaderFactory.getTimelineReaderStrategy(conf, useHttps, connTimeout);
}
Example 7
Source File: MRToTezHelper.java    From spork with Apache License 2.0 4 votes vote down vote up 
public static TezConfiguration getDAGAMConfFromMRConf(
        Configuration tezConf) {

    // Set Tez parameters based on MR parameters.
    TezConfiguration dagAMConf = new TezConfiguration(tezConf);
    Map<String, String> mrParamToDAGParamMap = DeprecatedKeys
            .getMRToDAGParamMap();

    for (Entry<String, String> entry : mrParamToDAGParamMap.entrySet()) {
        if (dagAMConf.get(entry.getKey()) != null) {
            dagAMConf.set(entry.getValue(), dagAMConf.get(entry.getKey()));
            dagAMConf.unset(entry.getKey());
            if (LOG.isDebugEnabled()) {
                LOG.debug("MR->DAG Translating MR key: " + entry.getKey()
                        + " to Tez key: " + entry.getValue()
                        + " with value " + dagAMConf.get(entry.getValue()));
            }
        }
    }

    String env = tezConf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV);
    if (tezConf.get(MRJobConfig.MR_AM_ENV) != null) {
        env = (env == null) ? tezConf.get(MRJobConfig.MR_AM_ENV)
                            : env + "," + tezConf.get(MRJobConfig.MR_AM_ENV);
    }

    if (env != null) {
        dagAMConf.setIfUnset(TezConfiguration.TEZ_AM_LAUNCH_ENV, env);
    }

    dagAMConf.setIfUnset(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS,
            org.apache.tez.mapreduce.hadoop.MRHelpers
                    .getJavaOptsForMRAM(tezConf));

    String queueName = tezConf.get(JobContext.QUEUE_NAME,
            YarnConfiguration.DEFAULT_QUEUE_NAME);
    dagAMConf.setIfUnset(TezConfiguration.TEZ_QUEUE_NAME, queueName);

    int amMemMB = tezConf.getInt(MRJobConfig.MR_AM_VMEM_MB,
            MRJobConfig.DEFAULT_MR_AM_VMEM_MB);
    int amCores = tezConf.getInt(MRJobConfig.MR_AM_CPU_VCORES,
            MRJobConfig.DEFAULT_MR_AM_CPU_VCORES);
    dagAMConf.setIfUnset(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, ""
            + amMemMB);
    dagAMConf.setIfUnset(TezConfiguration.TEZ_AM_RESOURCE_CPU_VCORES, ""
            + amCores);

    dagAMConf.setIfUnset(TezConfiguration.TEZ_AM_VIEW_ACLS,
            tezConf.get(MRJobConfig.JOB_ACL_VIEW_JOB, MRJobConfig.DEFAULT_JOB_ACL_VIEW_JOB));

    dagAMConf.setIfUnset(TezConfiguration.TEZ_AM_MODIFY_ACLS,
            tezConf.get(MRJobConfig.JOB_ACL_MODIFY_JOB, MRJobConfig.DEFAULT_JOB_ACL_MODIFY_JOB));

    dagAMConf.setIfUnset(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS, ""
            + dagAMConf.getInt(MRJobConfig.MR_AM_MAX_ATTEMPTS,
                    MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS));

    if (tezConf.get(MRConfiguration.JOB_CREDENTIALS_BINARY) != null) {
        dagAMConf.setIfUnset(TezConfiguration.TEZ_CREDENTIALS_PATH,
                tezConf.get(MRConfiguration.JOB_CREDENTIALS_BINARY));
    }

    //TODO: Strip out all MR settings

    return dagAMConf;
}
Example 8
Source File: CartesianProductConfig.java    From tez with Apache License 2.0 4 votes vote down vote up 
protected CartesianProductConfigProto toProto(TezConfiguration conf) {
  CartesianProductConfigProto.Builder builder =
    CartesianProductConfigProto.newBuilder();
  builder.setIsPartitioned(this.isPartitioned)
    .addAllSources(Arrays.asList(sources));

  if (isPartitioned) {
    builder.addAllNumPartitions(Ints.asList(numPartitions));
    if (filterDescriptor != null) {
      builder.setFilterClassName(filterDescriptor.getClassName());
      UserPayload filterUesrPayload = filterDescriptor.getUserPayload();
      if (filterUesrPayload != null) {
        builder.setFilterUserPayload(ByteString.copyFrom(filterUesrPayload.getPayload()));
      }
    }
  }

  if (conf != null) {
    builder.setMinFraction(conf.getFloat(
      CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_SLOW_START_MIN_FRACTION,
      CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_SLOW_START_MIN_FRACTION_DEFAULT));
    builder.setMaxFraction(conf.getFloat(
      CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_SLOW_START_MAX_FRACTION,
      CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_SLOW_START_MAX_FRACTION_DEFAULT));
    builder.setMaxParallelism(conf.getInt(
      CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_MAX_PARALLELISM,
      CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_MAX_PARALLELISM_DEFAULT));
    builder.setMinOpsPerWorker(conf.getLong(
      CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_MIN_OPS_PER_WORKER,
      CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_MIN_OPS_PER_WORKER_DEFAULT));
    builder.setEnableGrouping(conf.getBoolean(
      CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_ENABLE_GROUPING,
      CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_ENABLE_GROUPING_DEFAULT));
    if (conf.get(CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_GROUPING_FRACTION) != null) {
      builder.setGroupingFraction(Float.parseFloat(
        conf.get(CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_GROUPING_FRACTION)));
      Preconditions.checkArgument(0 < builder.getGroupingFraction() &&
        builder.getGroupingFraction() <= 1, "grouping fraction should be larger than 0 and less" +
        " or equal to 1, current value: " + builder.getGroupingFraction());
    }
    if (conf.get(CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_NUM_PARTITIONS) != null) {
      builder.setNumPartitionsForFairCase(Integer.parseInt(
        conf.get(CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_NUM_PARTITIONS)));
      Preconditions.checkArgument(builder.getNumPartitionsForFairCase() > 0,
        "Number of partitions for fair cartesian product should be positive integer");
    }
  }
  Preconditions.checkArgument(builder.getMinFraction() <= builder.getMaxFraction(),
    "min fraction(" + builder.getMinFraction() + ") should be less than max fraction(" +
      builder.getMaxFraction() + ") in cartesian product slow start");
  Preconditions.checkArgument(builder.getMaxParallelism() > 0,
    "max parallelism must be positive, currently is " + builder.getMaxParallelism());
  Preconditions.checkArgument(builder.getMinOpsPerWorker() > 0,
    "Min ops per worker must be positive, currently is " + builder.getMinOpsPerWorker());

  return builder.build();
}