Java Code Examples for org.apache.hadoop.mapred.JobConf#iterator()

@Nonnull
public static String toString(@Nonnull JobConf jobconf, @Nullable String regexKey) {
    final Iterator<Entry<String, String>> itor = jobconf.iterator();
    boolean hasNext = itor.hasNext();
    if (!hasNext) {
        return "";
    }
    final StringBuilder buf = new StringBuilder(1024);
    do {
        Entry<String, String> e = itor.next();
        hasNext = itor.hasNext();
        String k = e.getKey();
        if (k == null) {
            continue;
        }
        if (regexKey == null || k.matches(regexKey)) {
            String v = e.getValue();
            buf.append(k).append('=').append(v);
            if (hasNext) {
                buf.append(',');
            }
        }
    } while (hasNext);
    return buf.toString();
}
 

/**
 * Method to apply pig properties to JobConf (replaces properties with
 * resulting jobConf values).
 *
 * @param conf JobConf with appropriate hadoop resource files
 * @param properties Pig properties that will override hadoop properties;
 * properties might be modified
 */
public static void recomputeProperties(JobConf jobConf, Properties properties) {
    // We need to load the properties from the hadoop configuration
    // We want to override these with any existing properties we have.
    if (jobConf != null && properties != null) {
        // set user properties on the jobConf to ensure that defaults
        // and deprecation is applied correctly
        Enumeration<Object> propertiesIter = properties.keys();
        while (propertiesIter.hasMoreElements()) {
            String key = (String) propertiesIter.nextElement();
            String val = properties.getProperty(key);
            // We do not put user.name, See PIG-1419
            if (!key.equals("user.name")) {
                jobConf.set(key, val);
            }
        }
        // clear user defined properties and re-populate
        properties.clear();
        Iterator<Map.Entry<String, String>> iter = jobConf.iterator();
        while (iter.hasNext()) {
            Map.Entry<String, String> entry = iter.next();
            properties.put(entry.getKey(), entry.getValue());
        }
    }
}
 

void addJobConfToEnvironment(JobConf conf, Properties env) {
  if (debug_) {
    logprintln("addJobConfToEnvironment: begin");
  }
  Iterator it = conf.iterator();
  while (it.hasNext()) {
    Map.Entry en = (Map.Entry) it.next();
    String name = (String) en.getKey();
    //String value = (String)en.getValue(); // does not apply variable expansion
    String value = conf.get(name); // does variable expansion
    name = safeEnvVarName(name);
    envPut(env, name, value);
  }
  if (debug_) {
    logprintln("addJobConfToEnvironment: end");
  }
}
 

void addJobConfToEnvironment(JobConf conf, Properties env) {
  if (debug_) {
    logprintln("addJobConfToEnvironment: begin");
  }
  Iterator it = conf.iterator();
  while (it.hasNext()) {
    Map.Entry en = (Map.Entry) it.next();
    String name = (String) en.getKey();
    //String value = (String)en.getValue(); // does not apply variable expansion
    String value = conf.get(name); // does variable expansion 
    name = safeEnvVarName(name);
    envPut(env, name, value);
  }
  if (debug_) {
    logprintln("addJobConfToEnvironment: end");
  }
}
 

void addJobConfToEnvironment(JobConf conf, Properties env) {
  if (debug_) {
    logprintln("addJobConfToEnvironment: begin");
  }
  Iterator it = conf.iterator();
  while (it.hasNext()) {
    Map.Entry en = (Map.Entry) it.next();
    String name = (String) en.getKey();
    //String value = (String)en.getValue(); // does not apply variable expansion
    String value = conf.get(name); // does variable expansion 
    name = safeEnvVarName(name);
    envPut(env, name, value);
  }
  if (debug_) {
    logprintln("addJobConfToEnvironment: end");
  }
}
 

void addJobConfToEnvironment(JobConf jobconf, Properties env) {
  JobConf conf = new JobConf(jobconf);
  conf.setDeprecatedProperties();
  Iterator it = conf.iterator();
  while (it.hasNext()) {
    Map.Entry en = (Map.Entry) it.next();
    String name = (String) en.getKey();
    //String value = (String)en.getValue(); // does not apply variable expansion
    String value = conf.get(name); // does variable expansion 
    name = safeEnvVarName(name);
    envPut(env, name, value);
  }
}
 

void addJobConfToEnvironment(JobConf jobconf, Properties env) {
  JobConf conf = new JobConf(jobconf);
  conf.setDeprecatedProperties();
  Iterator it = conf.iterator();
  while (it.hasNext()) {
    Map.Entry en = (Map.Entry) it.next();
    String name = (String) en.getKey();
    //String value = (String)en.getValue(); // does not apply variable expansion
    String value = conf.get(name); // does variable expansion 
    name = safeEnvVarName(name);
    envPut(env, name, value);
  }
}
 

public JobConf getS3Conf() throws ExecException {
    JobConf jc = new JobConf();
    jc.addResource(CORE_SITE);
    Iterator<Entry<String, String>> i = jc.iterator();
    while (i.hasNext()) {
        Entry<String, String> e = i.next();
        String key = e.getKey();
        String value = e.getValue();
        if (key.startsWith("fs.s3") || key.startsWith("fs.s3n")) {
            jc.set(key, value);
        }
    }
    return jc;
}