Java Code Examples for org.apache.hadoop.mapreduce.v2.util.MRApps#setClassLoader()

The following examples show how to use org.apache.hadoop.mapreduce.v2.util.MRApps#setClassLoader() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: MRAppMaster.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
/**
 * Executes the given action with the job classloader set as the configuration
 * classloader as well as the thread context class loader if the job
 * classloader is enabled. After the call, the original classloader is
 * restored.
 *
 * If the job classloader is enabled and the code needs to load user-supplied
 * classes via configuration or thread context classloader, this method should
 * be used in order to load them.
 *
 * @param conf the configuration on which the classloader will be set
 * @param action the callable action to be executed
 */
<T> T callWithJobClassLoader(Configuration conf, Action<T> action) {
  // if the job classloader is enabled, we may need it to load the (custom)
  // classes; we make the job classloader available and unset it once it is
  // done
  ClassLoader currentClassLoader = conf.getClassLoader();
  boolean setJobClassLoader =
      jobClassLoader != null && currentClassLoader != jobClassLoader;
  if (setJobClassLoader) {
    MRApps.setClassLoader(jobClassLoader, conf);
  }
  try {
    return action.call(conf);
  } finally {
    if (setJobClassLoader) {
      // restore the original classloader
      MRApps.setClassLoader(currentClassLoader, conf);
    }
  }
}
Example 2
Source File: MRAppMaster.java    From big-c with Apache License 2.0 6 votes vote down vote up 
/**
 * Executes the given action with the job classloader set as the configuration
 * classloader as well as the thread context class loader if the job
 * classloader is enabled. After the call, the original classloader is
 * restored.
 *
 * If the job classloader is enabled and the code needs to load user-supplied
 * classes via configuration or thread context classloader, this method should
 * be used in order to load them.
 *
 * @param conf the configuration on which the classloader will be set
 * @param action the callable action to be executed
 */
<T> T callWithJobClassLoader(Configuration conf, Action<T> action) {
  // if the job classloader is enabled, we may need it to load the (custom)
  // classes; we make the job classloader available and unset it once it is
  // done
  ClassLoader currentClassLoader = conf.getClassLoader();
  boolean setJobClassLoader =
      jobClassLoader != null && currentClassLoader != jobClassLoader;
  if (setJobClassLoader) {
    MRApps.setClassLoader(jobClassLoader, conf);
  }
  try {
    return action.call(conf);
  } finally {
    if (setJobClassLoader) {
      // restore the original classloader
      MRApps.setClassLoader(currentClassLoader, conf);
    }
  }
}
Example 3
Source File: MRAppMaster.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@SuppressWarnings("unchecked")
@Override
protected void serviceStart() throws Exception {

  amInfos = new LinkedList<AMInfo>();
  completedTasksFromPreviousRun = new HashMap<TaskId, TaskInfo>();
  processRecovery();

  // Current an AMInfo for the current AM generation.
  AMInfo amInfo =
      MRBuilderUtils.newAMInfo(appAttemptID, startTime, containerID, nmHost,
          nmPort, nmHttpPort);

  // /////////////////// Create the job itself.
  job = createJob(getConfig(), forcedState, shutDownMessage);

  // End of creating the job.

  // Send out an MR AM inited event for all previous AMs.
  for (AMInfo info : amInfos) {
    dispatcher.getEventHandler().handle(
        new JobHistoryEvent(job.getID(), new AMStartedEvent(info
            .getAppAttemptId(), info.getStartTime(), info.getContainerId(),
            info.getNodeManagerHost(), info.getNodeManagerPort(), info
                .getNodeManagerHttpPort(), appSubmitTime)));
  }

  // Send out an MR AM inited event for this AM.
  dispatcher.getEventHandler().handle(
      new JobHistoryEvent(job.getID(), new AMStartedEvent(amInfo
          .getAppAttemptId(), amInfo.getStartTime(), amInfo.getContainerId(),
          amInfo.getNodeManagerHost(), amInfo.getNodeManagerPort(), amInfo
              .getNodeManagerHttpPort(), this.forcedState == null ? null
                  : this.forcedState.toString(), appSubmitTime)));
  amInfos.add(amInfo);

  // metrics system init is really init & start.
  // It's more test friendly to put it here.
  DefaultMetricsSystem.initialize("MRAppMaster");

  boolean initFailed = false;
  if (!errorHappenedShutDown) {
    // create a job event for job intialization
    JobEvent initJobEvent = new JobEvent(job.getID(), JobEventType.JOB_INIT);
    // Send init to the job (this does NOT trigger job execution)
    // This is a synchronous call, not an event through dispatcher. We want
    // job-init to be done completely here.
    jobEventDispatcher.handle(initJobEvent);

    // If job is still not initialized, an error happened during
    // initialization. Must complete starting all of the services so failure
    // events can be processed.
    initFailed = (((JobImpl)job).getInternalState() != JobStateInternal.INITED);

    // JobImpl's InitTransition is done (call above is synchronous), so the
    // "uber-decision" (MR-1220) has been made.  Query job and switch to
    // ubermode if appropriate (by registering different container-allocator
    // and container-launcher services/event-handlers).

    if (job.isUber()) {
      speculatorEventDispatcher.disableSpeculation();
      LOG.info("MRAppMaster uberizing job " + job.getID()
          + " in local container (\"uber-AM\") on node "
          + nmHost + ":" + nmPort + ".");
    } else {
      // send init to speculator only for non-uber jobs. 
      // This won't yet start as dispatcher isn't started yet.
      dispatcher.getEventHandler().handle(
          new SpeculatorEvent(job.getID(), clock.getTime()));
      LOG.info("MRAppMaster launching normal, non-uberized, multi-container "
          + "job " + job.getID() + ".");
    }
    // Start ClientService here, since it's not initialized if
    // errorHappenedShutDown is true
    clientService.start();
  }
  //start all the components
  super.serviceStart();

  // finally set the job classloader
  MRApps.setClassLoader(jobClassLoader, getConfig());

  if (initFailed) {
    JobEvent initFailedEvent = new JobEvent(job.getID(), JobEventType.JOB_INIT_FAILED);
    jobEventDispatcher.handle(initFailedEvent);
  } else {
    // All components have started, start the job.
    startJobs();
  }
}
Example 4
Source File: MRAppMaster.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@SuppressWarnings("unchecked")
@Override
protected void serviceStart() throws Exception {

  amInfos = new LinkedList<AMInfo>();
  completedTasksFromPreviousRun = new HashMap<TaskId, TaskInfo>();
  processRecovery();

  // Current an AMInfo for the current AM generation.
  AMInfo amInfo =
      MRBuilderUtils.newAMInfo(appAttemptID, startTime, containerID, nmHost,
          nmPort, nmHttpPort);

  // /////////////////// Create the job itself.
  job = createJob(getConfig(), forcedState, shutDownMessage);

  // End of creating the job.

  // Send out an MR AM inited event for all previous AMs.
  for (AMInfo info : amInfos) {
    dispatcher.getEventHandler().handle(
        new JobHistoryEvent(job.getID(), new AMStartedEvent(info
            .getAppAttemptId(), info.getStartTime(), info.getContainerId(),
            info.getNodeManagerHost(), info.getNodeManagerPort(), info
                .getNodeManagerHttpPort(), appSubmitTime)));
  }

  // Send out an MR AM inited event for this AM.
  dispatcher.getEventHandler().handle(
      new JobHistoryEvent(job.getID(), new AMStartedEvent(amInfo
          .getAppAttemptId(), amInfo.getStartTime(), amInfo.getContainerId(),
          amInfo.getNodeManagerHost(), amInfo.getNodeManagerPort(), amInfo
              .getNodeManagerHttpPort(), this.forcedState == null ? null
                  : this.forcedState.toString(), appSubmitTime)));
  amInfos.add(amInfo);

  // metrics system init is really init & start.
  // It's more test friendly to put it here.
  DefaultMetricsSystem.initialize("MRAppMaster");

  boolean initFailed = false;
  if (!errorHappenedShutDown) {
    // create a job event for job intialization
    JobEvent initJobEvent = new JobEvent(job.getID(), JobEventType.JOB_INIT);
    // Send init to the job (this does NOT trigger job execution)
    // This is a synchronous call, not an event through dispatcher. We want
    // job-init to be done completely here.
    jobEventDispatcher.handle(initJobEvent);

    // If job is still not initialized, an error happened during
    // initialization. Must complete starting all of the services so failure
    // events can be processed.
    initFailed = (((JobImpl)job).getInternalState() != JobStateInternal.INITED);

    // JobImpl's InitTransition is done (call above is synchronous), so the
    // "uber-decision" (MR-1220) has been made.  Query job and switch to
    // ubermode if appropriate (by registering different container-allocator
    // and container-launcher services/event-handlers).

    if (job.isUber()) {
      speculatorEventDispatcher.disableSpeculation();
      LOG.info("MRAppMaster uberizing job " + job.getID()
          + " in local container (\"uber-AM\") on node "
          + nmHost + ":" + nmPort + ".");
    } else {
      // send init to speculator only for non-uber jobs. 
      // This won't yet start as dispatcher isn't started yet.
      dispatcher.getEventHandler().handle(
          new SpeculatorEvent(job.getID(), clock.getTime()));
      LOG.info("MRAppMaster launching normal, non-uberized, multi-container "
          + "job " + job.getID() + ".");
    }
    // Start ClientService here, since it's not initialized if
    // errorHappenedShutDown is true
    clientService.start();
  }
  //start all the components
  super.serviceStart();

  // finally set the job classloader
  MRApps.setClassLoader(jobClassLoader, getConfig());

  if (initFailed) {
    JobEvent initFailedEvent = new JobEvent(job.getID(), JobEventType.JOB_INIT_FAILED);
    jobEventDispatcher.handle(initFailedEvent);
  } else {
    // All components have started, start the job.
    startJobs();
  }
}