Java Code Examples for org.apache.tez.dag.api.TezConfiguration#getBoolean()

public TezJob(TezConfiguration conf, DAG dag,
        Map<String, LocalResource> requestAMResources,
        int estimatedTotalParallelism) throws IOException {
    this.conf = conf;
    this.dag = dag;
    this.requestAMResources = requestAMResources;
    this.reuseSession = conf.getBoolean(PigConfiguration.PIG_TEZ_SESSION_REUSE, true);
    this.statusGetOpts = EnumSet.of(StatusGetOpts.GET_COUNTERS);
    tezJobConf = new TezJobConfig(estimatedTotalParallelism);
}
 

@Override
public void init(TezConfiguration tezConf, YarnConfiguration yarnConf) {
  this.conf = tezConf;
  // Tez libs already in the client's classpath
  this.conf.setBoolean(TezConfiguration.TEZ_IGNORE_LIB_URIS, true);
  this.conf.set(TezConfiguration.TEZ_AM_DAG_SCHEDULER_CLASS, localModeDAGSchedulerClassName);
  isSession = tezConf.getBoolean(TezConfiguration.TEZ_AM_SESSION_MODE,
      TezConfiguration.TEZ_AM_SESSION_MODE_DEFAULT);

  // disable web service for local mode.
  this.conf.setBoolean(TezConfiguration.TEZ_AM_WEBSERVICE_ENABLE, false);
}
 

public static FrameworkClient createFrameworkClient(TezConfiguration tezConf) {

    boolean isLocal = tezConf.getBoolean(TezConfiguration.TEZ_LOCAL_MODE, TezConfiguration.TEZ_LOCAL_MODE_DEFAULT);
    if (isLocal) {
      try {
        return ReflectionUtils.createClazzInstance("org.apache.tez.client.LocalClient");
      } catch (TezReflectionException e) {
        throw new TezUncheckedException("Fail to create LocalClient", e);
      }
    }
    return new TezYarnClient(YarnClient.createYarnClient());
  }
 

@Private
TezClient(String name, TezConfiguration tezConf,
          @Nullable Map<String, LocalResource> localResources,
          @Nullable Credentials credentials) {
  this(name, tezConf, tezConf.getBoolean(
      TezConfiguration.TEZ_AM_SESSION_MODE, TezConfiguration.TEZ_AM_SESSION_MODE_DEFAULT),
      localResources, credentials);
}
 

public DAGClientImpl(ApplicationId appId, String dagId, TezConfiguration conf,
    YarnConfiguration yarnConf, @Nullable FrameworkClient frameworkClient,
    UserGroupInformation ugi) {
  this.appId = appId;
  this.dagId = dagId;
  this.conf = conf;
  if (frameworkClient != null) {
    this.frameworkClient = frameworkClient;
  } else {
    this.frameworkClient = FrameworkClient.createFrameworkClient(conf);
    this.frameworkClient.init(conf, yarnConf);
    this.frameworkClient.start();
    cleanupFrameworkClient = true;
  }
  isATSEnabled = conf.get(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, "")
      .equals("org.apache.tez.dag.history.logging.ats.ATSHistoryLoggingService") &&
      conf.getBoolean(TezConfiguration.TEZ_DAG_HISTORY_LOGGING_ENABLED,
          TezConfiguration.TEZ_DAG_HISTORY_LOGGING_ENABLED_DEFAULT) &&
      conf.getBoolean(TezConfiguration.TEZ_AM_HISTORY_LOGGING_ENABLED,
          TezConfiguration.TEZ_AM_HISTORY_LOGGING_ENABLED_DEFAULT) &&
      DAGClientTimelineImpl.isSupported();

  realClient = new DAGClientRPCImpl(appId, dagId, conf, this.frameworkClient, ugi);
  statusPollInterval = conf.getLong(
      TezConfiguration.TEZ_DAG_STATUS_POLLINTERVAL_MS,
      TezConfiguration.TEZ_DAG_STATUS_POLLINTERVAL_MS_DEFAULT);
  if(statusPollInterval < 0) {
    LOG.error("DAG Status poll interval cannot be negative and setting to default value.");
    statusPollInterval = TezConfiguration.TEZ_DAG_STATUS_POLLINTERVAL_MS_DEFAULT;
  }
  this.diagnoticsWaitTimeout = conf.getLong(
      TezConfiguration.TEZ_CLIENT_DIAGNOSTICS_WAIT_TIMEOUT_MS,
      TezConfiguration.TEZ_CLIENT_DIAGNOSTICS_WAIT_TIMEOUT_MS_DEFAULT);
}
 

@Override
protected int runJob(String[] args, TezConfiguration tezConf,
    TezClient tezClient) throws Exception {
  System.out.println("Running SimpleSessionExample");
  String[] inputPaths = args[0].split(",");
  String[] outputPaths = args[1].split(",");
  if (inputPaths.length != outputPaths.length) {
    System.err.println("Inputs and outputs must be equal in number");
    return 3;
  }
  int numPartitions = args.length == 3 ? Integer.parseInt(args[2]) : 1;

  // Session pre-warming allows the user to hide initial startup, resource acquisition latency etc.
  // by pre-allocating execution resources in the Tez session. They can run initialization logic 
  // in these pre-allocated resources (containers) to pre-warm the containers.
  // In between DAG executions, the session can hold on to a minimum number of containers.
  // Ideally, this would be enough to provide desired balance of efficiency for the application 
  // and sharing of resources with other applications. Typically, the number of containers to be 
  // pre-warmed equals the number of containers to be held between DAGs.
  if (tezConf.getBoolean(enablePrewarmConfig, false)) {
    // the above parameter is not a Tez parameter. Its only for this example.
    // In this example we are pre-warming enough containers to run all the sum tasks in parallel.
    // This means pre-warming numPartitions number of containers.
    // We are making the pre-warm and held containers to be the same and using the helper API to 
    // set up pre-warming. They can be made different and also custom initialization logic can be 
    // specified using other API's. We know that the OrderedWordCount dag uses default files and 
    // resources. Otherwise we would have to specify matching parameters in the preWarm API too.
    tezConf.setInt(TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS, numPartitions);
    tezClient.preWarm(PreWarmVertex.createConfigBuilder(tezConf).build());
  }

  for (int i = 0; i < inputPaths.length; ++i) {
    DAG dag = OrderedWordCount.createDAG(tezConf, inputPaths[i], outputPaths[i], numPartitions,
        isDisableSplitGrouping(), isGenerateSplitInClient(), ("DAG-Iteration-" + i)); // the names of the DAGs must be unique in a session

    LOG.info("Running dag number " + i);
    if(runDag(dag, isCountersLog(), LOG) != 0) {
      return -1;
    }
  }
  return 0;
}
 

/**
 * Setup LocalResource map for Tez jars based on provided Configuration
 * 
 * @param conf
 *          Configuration to use to access Tez jars' locations
 * @param credentials
 *          a credentials instance into which tokens for the Tez local
 *          resources will be populated
 * @param tezJarResources Map of LocalResources to use for AM and DAGs
 * @return Whether the archive-based deployment of Tez was used.
 * @throws IOException
 */
static boolean setupTezJarsLocalResources(TezConfiguration conf,
    Credentials credentials, Map<String, LocalResource> tezJarResources)
    throws IOException {
  Objects.requireNonNull(credentials, "A non-null credentials object should be specified");
  boolean usingTezArchive = false;

  if (conf.getBoolean(TezConfiguration.TEZ_IGNORE_LIB_URIS, false)){
    LOG.info("Ignoring '" + TezConfiguration.TEZ_LIB_URIS + "' since  '" + 
          TezConfiguration.TEZ_IGNORE_LIB_URIS + "' is set to true");
  } else {
    // Add tez jars to local resource
    String[] tezJarUris = conf.getStrings(TezConfiguration.TEZ_LIB_URIS);

    if (tezJarUris == null || tezJarUris.length == 0) {
      throw new TezUncheckedException("Invalid configuration of tez jars"
          + ", " + TezConfiguration.TEZ_LIB_URIS
          + " is not defined in the configuration");
    }

    LOG.info("Using tez.lib.uris value from configuration: "
        + conf.get(TezConfiguration.TEZ_LIB_URIS));
    LOG.info("Using tez.lib.uris.classpath value from configuration: "
        + conf.get(TezConfiguration.TEZ_LIB_URIS_CLASSPATH));

    usingTezArchive = addLocalResources(conf, tezJarUris,
                       tezJarResources, credentials);

    if (tezJarResources.isEmpty()) {
      throw new TezUncheckedException(
          "No files found in locations specified in "
              + TezConfiguration.TEZ_LIB_URIS + " . Locations: "
              + StringUtils.join(tezJarUris, ','));
    }
  }

  // Add aux uris to local resources
  addLocalResources(conf, conf.getStrings(TezConfiguration.TEZ_AUX_URIS),
      tezJarResources, credentials);

  return usingTezArchive;
}
 

private TezClient(String name, TezConfiguration tezConf) {
  this(name, tezConf, tezConf.getBoolean(
      TezConfiguration.TEZ_AM_SESSION_MODE, TezConfiguration.TEZ_AM_SESSION_MODE_DEFAULT));    
}
 

/**
 * Create a new TezClient. Session or non-session execution mode will be
 * inferred from configuration. Set the initial resources and security
 * credentials for the App Master. If app master resources/credentials are
 * needed then this is the recommended constructor for session mode execution.
 * 
 * @param name
 *          Name of the client. Used for logging etc. This will also be used
 *          as app master name is session mode
 * @param tezConf
 *          Configuration for the framework
 * @param localResources
 *          resources for the App Master
 * @param credentials
 *          Set security credentials to be used inside the app master, if
 *          needed. Tez App Master needs credentials to access the staging
 *          directory and for most HDFS cases these are automatically obtained
 *          by Tez client. If the staging directory is on a file system for
 *          which credentials cannot be obtained or for any credentials needed
 *          by user code running inside the App Master, credentials must be
 *          supplied by the user. These will be used by the App Master for the
 *          next DAG. <br>
 *          In session mode, credentials, if needed, must be set before
 *          calling start()
 */
public TezClient(String name, TezConfiguration tezConf,
    @Nullable Map<String, LocalResource> localResources,
    @Nullable Credentials credentials) {
  this(name, tezConf, tezConf.getBoolean(
      TezConfiguration.TEZ_AM_SESSION_MODE, TezConfiguration.TEZ_AM_SESSION_MODE_DEFAULT),
      localResources, credentials);
}
 

/**
 * Create an instance of a TezClientBuilder
 *
 * @param name
 *          Name of the client. Used for logging etc. This will also be used
 *          as app master name is session mode
 * @param tezConf
 *          Configuration for the framework
 */
private TezClientBuilder(String name, TezConfiguration tezConf) {
  this.name = name;
  this.tezConf = tezConf;
  isSession = tezConf.getBoolean(
      TezConfiguration.TEZ_AM_SESSION_MODE, TezConfiguration.TEZ_AM_SESSION_MODE_DEFAULT);
}
 

/**
 * Create a new TezClient. Session or non-session execution mode will be
 * inferred from configuration. 
 * @param name
 *          Name of the client. Used for logging etc. This will also be used
 *          as app master name is session mode
 * @param tezConf
 *          Configuration for the framework
 */
public TezClient(String name, TezConfiguration tezConf) {
  this(name, tezConf, tezConf.getBoolean(
      TezConfiguration.TEZ_AM_SESSION_MODE, TezConfiguration.TEZ_AM_SESSION_MODE_DEFAULT));    
}