Java Code Examples for org.apache.hadoop.conf.Configuration#getClassByName()

public static Class<?> getClassFromConfig(Configuration configuration, String configName, Class<?> assignableFrom) {
  final String className = configuration.get(configName);
  if (className == null) {
    return null;
  }
  
  try {
    final Class<?> foundClass = configuration.getClassByName(className);	
    if (!assignableFrom.isAssignableFrom(foundClass)) {
      throw new BadConfigurationException("class " + className + " set in job conf at "
              + configName + " is not a subclass of " + assignableFrom.getCanonicalName());
    }
    return foundClass;
  } catch (ClassNotFoundException e) {
    throw new BadConfigurationException("could not instantiate class " + className + " set in job conf at " + configName, e);
  }
}
 

/**
 * Returns a {@link List} containing <em>all</em> of the topic-group-{@link Mapper} combinations added via
 * {@link #addTopic(Job, String, String, Class)}.
 * 
 * @param conf
 *            the conf for this job.
 * @return all of the configured {@link TopicConf}s
 */
@SuppressWarnings("unchecked")
public static List<TopicConf> getTopics(final Configuration conf) {
    final List<TopicConf> result = Lists.newArrayList();
    for (final String topicConf : conf.get(TOPICS_CONF).split(";")) {
        final String[] topicConfTokens = topicConf.split(",");
        final String topic = topicConfTokens[0];
        final String group = topicConfTokens[1];
        final Class<? extends Mapper> mapper;
        try {
            mapper = (Class<? extends Mapper>) conf.getClassByName(topicConfTokens[2]);
        } catch (final ClassNotFoundException e) {
            throw new RuntimeException(e);
        }
        result.add(new TopicConf(topic, group, mapper));
    }
    return result;
}
 

private synchronized void initializeClusterManager(Configuration conf, ClusterType clusterType)
    throws ClusterManagerInitilizationException
{
  if (clusterManager != null) {
    return;
  }

  String clusterManagerClassName = CacheConfig.getClusterManagerClass(conf, clusterType);
  log.info("Initializing cluster manager : " + clusterManagerClassName);

  try {
    Class clusterManagerClass = conf.getClassByName(clusterManagerClassName);
    Constructor constructor = clusterManagerClass.getConstructor();
    ClusterManager manager = (ClusterManager) constructor.newInstance();

    manager.initialize(conf);
    setClusterManager(manager);
  }
  catch (ClassNotFoundException | NoSuchMethodException | InstantiationException |
          IllegalAccessException | InvocationTargetException ex) {
    String errorMessage = String.format("Not able to initialize ClusterManager class : {0} ",
        clusterManagerClassName);
    log.error(errorMessage, ex);
    throw new ClusterManagerInitilizationException(errorMessage, ex);
  }
}
 

static Class<?> getProtocolClass(String protocolName, Configuration conf) 
throws ClassNotFoundException {
  Class<?> protocol = PROTOCOL_CACHE.get(protocolName);
  if (protocol == null) {
    protocol = conf.getClassByName(protocolName);
    PROTOCOL_CACHE.put(protocolName, protocol);
  }
  return protocol;
}
 

@SuppressWarnings("unchecked")
private void add(Configuration conf, String serializationName) {
  try {
    
    Class<? extends Serialization> serializionClass =
      (Class<? extends Serialization>) conf.getClassByName(serializationName);
    serializations.add((Serialization)
        ReflectionUtils.newInstance(serializionClass, getConf()));
  } catch (ClassNotFoundException e) {
    LOG.warn("Serilization class not found: " +
        StringUtils.stringifyException(e));
  }
}
 

MRInputConfigBuilder(Configuration conf, Class<?> inputFormatParam) {
  this.conf = conf;
  if (inputFormatParam != null) {
    inputFormatProvided = true;
    this.inputFormat = inputFormatParam;
    if (org.apache.hadoop.mapred.InputFormat.class.isAssignableFrom(inputFormatParam)) {
      useNewApi = false;
    } else if (org.apache.hadoop.mapreduce.InputFormat.class.isAssignableFrom(inputFormatParam)) {
      useNewApi = true;
    } else {
      throw new TezUncheckedException("inputFormat must be assignable from either " +
          "org.apache.hadoop.mapred.InputFormat or " +
          "org.apache.hadoop.mapreduce.InputFormat" +
          " Given: " + inputFormatParam.getName());
    }
  } else {
    inputFormatProvided = false;
    useNewApi = conf.getBoolean(MRJobConfig.NEW_API_MAPPER_CONFIG, true);
    try {
      if (useNewApi) {
        this.inputFormat = conf.getClassByName(conf.get(MRJobConfig.INPUT_FORMAT_CLASS_ATTR));
        Preconditions.checkState(org.apache.hadoop.mapreduce.InputFormat.class
            .isAssignableFrom(this.inputFormat));
      } else {
        this.inputFormat = conf.getClassByName(conf.get("mapred.input.format.class"));
        Preconditions.checkState(org.apache.hadoop.mapred.InputFormat.class
            .isAssignableFrom(this.inputFormat));
      }
    } catch (ClassNotFoundException e) {
      throw new TezUncheckedException(e);
    }
    initializeInputPath();
  }
}
 

@SuppressWarnings("unchecked")
private void add(Configuration conf, String serializationName) {
  try {
    Class<? extends Serialization> serializionClass =
      (Class<? extends Serialization>) conf.getClassByName(serializationName);
    serializations.add((Serialization)
    ReflectionUtils.newInstance(serializionClass, getConf()));
  } catch (ClassNotFoundException e) {
    LOG.warn("Serialization class not found: ", e);
  }
}
 

/**
 * Get the socket factory corresponding to the given proxy URI. If the
 * given proxy URI corresponds to an absence of configuration parameter,
 * returns null. If the URI is malformed raises an exception.
 * 
 * @param propValue the property which is the class name of the
 *        SocketFactory to instantiate; assumed non null and non empty.
 * @return a socket factory as defined in the property value.
 */
public static SocketFactory getSocketFactoryFromProperty(
    Configuration conf, String propValue) {

  try {
    Class<?> theClass = conf.getClassByName(propValue);
    return (SocketFactory) ReflectionUtils.newInstance(theClass, conf);

  } catch (ClassNotFoundException cnfe) {
    throw new RuntimeException("Socket Factory class not found: " + cnfe);
  }
}
 

@VisibleForTesting
public static String getAccountKeyFromConfiguration(String accountName,
    Configuration conf) throws KeyProviderException {
  String key = null;
  String keyProviderClass = conf.get(KEY_ACCOUNT_KEYPROVIDER_PREFIX
      + accountName);
  KeyProvider keyProvider = null;

  if (keyProviderClass == null) {
    // No key provider was provided so use the provided key as is.
    keyProvider = new SimpleKeyProvider();
  } else {
    // create an instance of the key provider class and verify it
    // implements KeyProvider
    Object keyProviderObject = null;
    try {
      Class<?> clazz = conf.getClassByName(keyProviderClass);
      keyProviderObject = clazz.newInstance();
    } catch (Exception e) {
      throw new KeyProviderException("Unable to load key provider class.", e);
    }
    if (!(keyProviderObject instanceof KeyProvider)) {
      throw new KeyProviderException(keyProviderClass
          + " specified in config is not a valid KeyProvider class.");
    }
    keyProvider = (KeyProvider) keyProviderObject;
  }
  key = keyProvider.getStorageAccountKey(accountName, conf);

  return key;
}
 

/** It may seem strange to silently switch behaviour when a String
 * is not a classname; the reason is simplified Usage:<pre>
 * -mapper [classname | program ]
 * instead of the explicit Usage:
 * [-mapper program | -javamapper classname], -mapper and -javamapper are mutually exclusive.
 * (repeat for -reducer, -combiner) </pre>
 */
public static Class goodClassOrNull(Configuration conf, String className, String defaultPackage) {
  if (className.indexOf('.') == -1 && defaultPackage != null) {
    className = defaultPackage + "." + className;
  }
  Class clazz = null;
  try {
    clazz = conf.getClassByName(className);
  } catch (ClassNotFoundException cnf) {
  }
  return clazz;
}
 

/** Return the class for a name.  Default is {@link Class#forName(String)}.*/
public static synchronized Class<?> getClass(String name, Configuration conf
                                          ) throws IOException {
  Class<?> writableClass = NAME_TO_CLASS.get(name);
  if (writableClass != null)
    return writableClass.asSubclass(Writable.class);
  try {
    return conf.getClassByName(name);
  } catch (ClassNotFoundException e) {
    IOException newE = new IOException("WritableName can't load class: " + name);
    newE.initCause(e);
    throw newE;
  }
}
 

@SuppressWarnings("unchecked")
private void add(Configuration conf, String serializationName) {
  try {
    Class<? extends Serialization> serializionClass =
      (Class<? extends Serialization>) conf.getClassByName(serializationName);
    serializations.add((Serialization)
    ReflectionUtils.newInstance(serializionClass, getConf()));
  } catch (ClassNotFoundException e) {
    LOG.warn("Serialization class not found: ", e);
  }
}
 

/** Return the class for a name.  Default is {@link Class#forName(String)}.*/
public static synchronized Class<?> getClass(String name, Configuration conf
                                          ) throws IOException {
  Class<?> writableClass = NAME_TO_CLASS.get(name);
  if (writableClass != null)
    return writableClass.asSubclass(Writable.class);
  try {
    return conf.getClassByName(name);
  } catch (ClassNotFoundException e) {
    IOException newE = new IOException("WritableName can't load class: " + name);
    newE.initCause(e);
    throw newE;
  }
}
 

/**
 * Get the list of codecs discovered via a Java ServiceLoader, or
 * listed in the configuration. Codecs specified in configuration come
 * later in the returned list, and are considered to override those
 * from the ServiceLoader.
 * @param conf the configuration to look in
 * @return a list of the {@link CompressionCodec} classes
 */
public static List<Class<? extends CompressionCodec>> getCodecClasses(
    Configuration conf) {
  List<Class<? extends CompressionCodec>> result
    = new ArrayList<Class<? extends CompressionCodec>>();
  // Add codec classes discovered via service loading
  synchronized (CODEC_PROVIDERS) {
    // CODEC_PROVIDERS is a lazy collection. Synchronize so it is
    // thread-safe. See HADOOP-8406.
    for (CompressionCodec codec : CODEC_PROVIDERS) {
      result.add(codec.getClass());
    }
  }
  // Add codec classes from configuration
  String codecsString = conf.get(
      CommonConfigurationKeys.IO_COMPRESSION_CODECS_KEY);
  if (codecsString != null) {
    StringTokenizer codecSplit = new StringTokenizer(codecsString, ",");
    while (codecSplit.hasMoreElements()) {
      String codecSubstring = codecSplit.nextToken().trim();
      if (codecSubstring.length() != 0) {
        try {
          Class<?> cls = conf.getClassByName(codecSubstring);
          if (!CompressionCodec.class.isAssignableFrom(cls)) {
            throw new IllegalArgumentException("Class " + codecSubstring +
                                               " is not a CompressionCodec");
          }
          result.add(cls.asSubclass(CompressionCodec.class));
        } catch (ClassNotFoundException ex) {
          throw new IllegalArgumentException("Compression codec " + 
                                             codecSubstring + " not found.",
                                             ex);
        }
      }
    }
  }
  return result;
}
 

static Class<?> getProtocolClass(String protocolName, Configuration conf) 
throws ClassNotFoundException {
  Class<?> protocol = PROTOCOL_CACHE.get(protocolName);
  if (protocol == null) {
    protocol = conf.getClassByName(protocolName);
    PROTOCOL_CACHE.put(protocolName, protocol);
  }
  return protocol;
}
 

/**
 * Get the socket factory corresponding to the given proxy URI. If the
 * given proxy URI corresponds to an absence of configuration parameter,
 * returns null. If the URI is malformed raises an exception.
 * 
 * @param propValue the property which is the class name of the
 *        SocketFactory to instantiate; assumed non null and non empty.
 * @return a socket factory as defined in the property value.
 */
public static SocketFactory getSocketFactoryFromProperty(
    Configuration conf, String propValue) {

  try {
    Class<?> theClass = conf.getClassByName(propValue);
    return (SocketFactory) ReflectionUtils.newInstance(theClass, conf);

  } catch (ClassNotFoundException cnfe) {
    throw new RuntimeException("Socket Factory class not found: " + cnfe);
  }
}
 

@SuppressWarnings("unchecked")
private void add(Configuration conf, String serializationName) {
  try {
    
    Class<? extends Serialization> serializionClass =
      (Class<? extends Serialization>) conf.getClassByName(serializationName);
    serializations.add((Serialization)
        ReflectionUtils.newInstance(serializionClass, getConf()));
  } catch (ClassNotFoundException e) {
    LOG.warn("Serilization class not found: " +
        StringUtils.stringifyException(e));
  }
}
 

private void runTest(final JobClient jc, final Configuration conf,
    final String jobClass, final String[] args, KillTaskThread killTaskThread,
    KillTrackerThread killTrackerThread) throws Exception {
  int prevJobsNum = jc.getAllJobs().length;
  Thread t = new Thread("Job Test") {
    public void run() {
      try {
        Class<?> jobClassObj = conf.getClassByName(jobClass);
        int status = ToolRunner.run(conf, (Tool)(jobClassObj.newInstance()), 
            args);
        checkJobExitStatus(status, jobClass);
      } catch (Exception e) {
        LOG.fatal("JOB " + jobClass + " failed to run");
        System.exit(-1);
      }
    }
  };
  t.setDaemon(true);
  t.start();
  JobStatus[] jobs;
  //get the job ID. This is the job that we just submitted
  while ((jobs = jc.getAllJobs()).length - prevJobsNum == 0) {
    LOG.info("Waiting for the job " + jobClass +" to start");
    Thread.sleep(1000);
  }
  JobID jobId = jobs[jobs.length - 1].getJobID();
  RunningJob rJob = jc.getJob(jobId);
  while (rJob.getJobState() == JobStatus.PREP) {
    LOG.info("JobID : " + jobId + " not started RUNNING yet");
    Thread.sleep(1000);
    rJob = jc.getJob(jobId);
  }
  if (killTaskThread != null) {
    killTaskThread.setRunningJob(rJob);
    killTaskThread.start();
    killTaskThread.join();
    LOG.info("DONE WITH THE TASK KILL/FAIL TESTS");
  }
  if (killTrackerThread != null) {
    killTrackerThread.setRunningJob(rJob);
    killTrackerThread.start();
    killTrackerThread.join();
    LOG.info("DONE WITH THE TESTS TO DO WITH LOST TASKTRACKERS");
  }
  t.join();
}
 

private void runTest(final JobClient jc, final Configuration conf,
    final String jobClass, final String[] args, KillTaskThread killTaskThread,
    KillTrackerThread killTrackerThread) throws Exception {
  Thread t = new Thread("Job Test") {
    public void run() {
      try {
        Class<?> jobClassObj = conf.getClassByName(jobClass);
        int status = ToolRunner.run(conf, (Tool)(jobClassObj.newInstance()), 
            args);
        checkJobExitStatus(status, jobClass);
      } catch (Exception e) {
        LOG.fatal("JOB " + jobClass + " failed to run");
        System.exit(-1);
      }
    }
  };
  t.setDaemon(true);
  t.start();
  JobStatus[] jobs;
  //get the job ID. This is the job that we just submitted
  while ((jobs = jc.jobsToComplete()).length == 0) {
    LOG.info("Waiting for the job " + jobClass +" to start");
    Thread.sleep(1000);
  }
  JobID jobId = jobs[jobs.length - 1].getJobID();
  RunningJob rJob = jc.getJob(jobId);
  if(rJob.isComplete()) {
    LOG.error("The last job returned by the querying JobTracker is complete :" + 
        rJob.getJobID() + " .Exiting the test");
    System.exit(-1);
  }
  while (rJob.getJobState() == JobStatus.PREP) {
    LOG.info("JobID : " + jobId + " not started RUNNING yet");
    Thread.sleep(1000);
    rJob = jc.getJob(jobId);
  }
  if (killTaskThread != null) {
    killTaskThread.setRunningJob(rJob);
    killTaskThread.start();
    killTaskThread.join();
    LOG.info("DONE WITH THE TASK KILL/FAIL TESTS");
  }
  if (killTrackerThread != null) {
    killTrackerThread.setRunningJob(rJob);
    killTrackerThread.start();
    killTrackerThread.join();
    LOG.info("DONE WITH THE TESTS TO DO WITH LOST TASKTRACKERS");
  }
  t.join();
}
 

private void runTest(final JobClient jc, final Configuration conf,
    final String jobClass, final String[] args, KillTaskThread killTaskThread,
    KillTrackerThread killTrackerThread) throws Exception {
  Thread t = new Thread("Job Test") {
    public void run() {
      try {
        Class<?> jobClassObj = conf.getClassByName(jobClass);
        int status = ToolRunner.run(conf, (Tool)(jobClassObj.newInstance()), 
            args);
        checkJobExitStatus(status, jobClass);
      } catch (Exception e) {
        LOG.fatal("JOB " + jobClass + " failed to run");
        System.exit(-1);
      }
    }
  };
  t.setDaemon(true);
  t.start();
  JobStatus[] jobs;
  //get the job ID. This is the job that we just submitted
  while ((jobs = jc.jobsToComplete()).length == 0) {
    LOG.info("Waiting for the job " + jobClass +" to start");
    Thread.sleep(1000);
  }
  JobID jobId = jobs[jobs.length - 1].getJobID();
  RunningJob rJob = jc.getJob(jobId);
  if(rJob.isComplete()) {
    LOG.error("The last job returned by the querying JobTracker is complete :" + 
        rJob.getJobID() + " .Exiting the test");
    System.exit(-1);
  }
  while (rJob.getJobState() == JobStatus.PREP) {
    LOG.info("JobID : " + jobId + " not started RUNNING yet");
    Thread.sleep(1000);
    rJob = jc.getJob(jobId);
  }
  if (killTaskThread != null) {
    killTaskThread.setRunningJob(rJob);
    killTaskThread.start();
    killTaskThread.join();
    LOG.info("DONE WITH THE TASK KILL/FAIL TESTS");
  }
  if (killTrackerThread != null) {
    killTrackerThread.setRunningJob(rJob);
    killTrackerThread.start();
    killTrackerThread.join();
    LOG.info("DONE WITH THE TESTS TO DO WITH LOST TASKTRACKERS");
  }
  t.join();
}