Java Code Examples for org.apache.hadoop.conf.Configuration#iterator()

public static Properties extractKafkaConfigToProperties(Configuration configuration) {
    Set<String> configNames = new HashSet<String>();
    try {
        configNames = ConsumerConfig.configNames();
    } catch (Exception e) {
        // the Kafka configNames api is supported on 0.10.1.0+, in case NoSuchMethodException which is an Error, not Exception
        String[] configNamesArray = ("metric.reporters, metadata.max.age.ms, partition.assignment.strategy, reconnect.backoff.ms," + "sasl.kerberos.ticket.renew.window.factor, max.partition.fetch.bytes, bootstrap.servers, ssl.keystore.type," + " enable.auto.commit, sasl.mechanism, interceptor.classes, exclude.internal.topics, ssl.truststore.password," + " client.id, ssl.endpoint.identification.algorithm, max.poll.records, check.crcs, request.timeout.ms, heartbeat.interval.ms," + " auto.commit.interval.ms, receive.buffer.bytes, ssl.truststore.type, ssl.truststore.location, ssl.keystore.password, fetch.min.bytes," + " fetch.max.bytes, send.buffer.bytes, max.poll.interval.ms, value.deserializer, group.id, retry.backoff.ms,"
                + " ssl.secure.random.implementation, sasl.kerberos.kinit.cmd, sasl.kerberos.service.name, sasl.kerberos.ticket.renew.jitter, ssl.trustmanager.algorithm, ssl.key.password, fetch.max.wait.ms, sasl.kerberos.min.time.before.relogin, connections.max.idle.ms, session.timeout.ms, metrics.num.samples, key.deserializer, ssl.protocol, ssl.provider, ssl.enabled.protocols, ssl.keystore.location, ssl.cipher.suites, security.protocol, ssl.keymanager.algorithm, metrics.sample.window.ms, auto.offset.reset").split(",");
        configNames.addAll(Arrays.asList(configNamesArray));
    }

    Properties result = new Properties();
    for (Iterator<Map.Entry<String, String>> it = configuration.iterator(); it.hasNext();) {
        Map.Entry<String, String> entry = it.next();
        String key = entry.getKey();
        String value = entry.getValue();
        if (configNames.contains(key)) {
            result.put(key, value);
        }
    }
    return result;
}
 

/**
 * Replaces the given entries in the configuration by clearing the
 * Configuration and re-adding the elements that aren't in the Map of
 * entries to unset
 * 
 * @param conf
 * @param entriesToUnset
 */
protected void clearUnset(Configuration conf,
        Map<String, String> entriesToUnset) {
    // Gets a copy of the entries
    Iterator<Entry<String, String>> originalEntries = conf.iterator();
    conf.clear();

    while (originalEntries.hasNext()) {
        Entry<String, String> originalEntry = originalEntries.next();

        // Only re-set() the pairs that aren't in our collection of keys to
        // unset
        if (!entriesToUnset.containsKey(originalEntry.getKey())) {
            conf.set(originalEntry.getKey(), originalEntry.getValue());
        }
    }
}
 

public static String convertToHistoryText(String description, Configuration conf) throws IOException {
    // Add a version if this serialization is changed
    JSONObject jsonObject = new JSONObject();
    try {
        if (description != null && !description.isEmpty()) {
            jsonObject.put("desc", description);
    }
    if (conf != null) {
        JSONObject confJson = new JSONObject();
        Iterator<Entry<String, String>> iter = conf.iterator();
        while (iter.hasNext()) {
            Entry<String, String> entry = iter.next();
            confJson.put(entry.getKey(), entry.getValue());
        }
        jsonObject.put("config", confJson);
    }
    } catch (JSONException e) {
        throw new IOException("Error when trying to convert description/conf to JSON", e);
    }
    return jsonObject.toString();
}
 

/**
 * Sets the desired configuration.
 */
@CliCommand(value = "configuration set", help = "Sets the desired configuration")
public String setConfig(@CliOption(key = "type", mandatory = true, help = "Type of the configuration") ConfigType configType,
                        @CliOption(key = "url", help = "URL of the config") String url,
                        @CliOption(key = "file", help = "File of the config") File file) throws IOException {
  Configuration configuration = new Configuration(false);
  if (file == null) {
    configuration.addResource(new URL(url));
  } else {
    configuration.addResource(new FileInputStream(file));
  }
  Map<String, String> config = new HashMap<String, String>();
  Iterator<Map.Entry<String, String>> iterator = configuration.iterator();
  while (iterator.hasNext()) {
    Map.Entry<String, String> entry = iterator.next();
    config.put(entry.getKey(), entry.getValue());
  }
  client.modifyConfiguration(configType.getName(), config);
  return "Restart is required!\n" + renderSingleMap(config, "KEY", "VALUE");
}
 

public static Properties extractKafkaConfigToProperties(Configuration configuration) {
    Set<String> configNames = new HashSet<String>();
    try {
        configNames = ConsumerConfig.configNames();
    } catch (Exception e) {
        // the Kafka configNames api is supported on 0.10.1.0+, in case NoSuchMethodException which is an Error, not Exception
        String[] configNamesArray = ("metric.reporters, metadata.max.age.ms, partition.assignment.strategy, reconnect.backoff.ms," + "sasl.kerberos.ticket.renew.window.factor, max.partition.fetch.bytes, bootstrap.servers, ssl.keystore.type," + " enable.auto.commit, sasl.mechanism, interceptor.classes, exclude.internal.topics, ssl.truststore.password," + " client.id, ssl.endpoint.identification.algorithm, max.poll.records, check.crcs, request.timeout.ms, heartbeat.interval.ms," + " auto.commit.interval.ms, receive.buffer.bytes, ssl.truststore.type, ssl.truststore.location, ssl.keystore.password, fetch.min.bytes," + " fetch.max.bytes, send.buffer.bytes, max.poll.interval.ms, value.deserializer, group.id, retry.backoff.ms,"
                + " ssl.secure.random.implementation, sasl.kerberos.kinit.cmd, sasl.kerberos.service.name, sasl.kerberos.ticket.renew.jitter, ssl.trustmanager.algorithm, ssl.key.password, fetch.max.wait.ms, sasl.kerberos.min.time.before.relogin, connections.max.idle.ms, session.timeout.ms, metrics.num.samples, key.deserializer, ssl.protocol, ssl.provider, ssl.enabled.protocols, ssl.keystore.location, ssl.cipher.suites, security.protocol, ssl.keymanager.algorithm, metrics.sample.window.ms, auto.offset.reset").split(",");
        configNames.addAll(Arrays.asList(configNamesArray));
    }

    Properties result = new Properties();
    for (Iterator<Map.Entry<String, String>> it = configuration.iterator(); it.hasNext();) {
        Map.Entry<String, String> entry = it.next();
        String key = entry.getKey();
        String value = entry.getValue();
        if (configNames.contains(key)) {
            result.put(key, value);
        }
    }
    return result;
}
 

/**
 * Utility to work around the limitation of the copy constructor
 * {@link Configuration#Configuration(Configuration)} provided by the {@link Configuration}
 * class. See https://issues.apache.org/jira/browse/HBASE-18378.
 * The copy constructor doesn't copy all the config settings, so we need to resort to
 * iterating through all the settings and setting it on the cloned config.
 * @param toCopy  configuration to copy
 * @return
 */
private static Configuration cloneConfig(Configuration toCopy) {
    Configuration clone = new Configuration();
    Iterator<Entry<String, String>> iterator = toCopy.iterator();
    while (iterator.hasNext()) {
        Entry<String, String> entry = iterator.next();
        clone.set(entry.getKey(), entry.getValue());
    }
    return clone;
}
 

/**
 * Returns the HBase {@code Put} instances to store for the given
 * {@code Configuration} data. Each configuration property will be stored as a
 * separate key value.
 *
 * @param jobDesc the {@link JobDesc} generated for the job
 * @param jobConf the job configuration
 * @return puts for the given job configuration
 */
public static List<Put> getHbasePuts(JobDesc jobDesc, Configuration jobConf) {
  List<Put> puts = new LinkedList<Put>();

  JobKey jobKey = new JobKey(jobDesc);
  byte[] jobKeyBytes = new JobKeyConverter().toBytes(jobKey);

  // Add all columns to one put
  Put jobPut = new Put(jobKeyBytes);
  jobPut.addColumn(Constants.INFO_FAM_BYTES, Constants.VERSION_COLUMN_BYTES,
      Bytes.toBytes(jobDesc.getVersion()));
  jobPut.addColumn(Constants.INFO_FAM_BYTES, Constants.FRAMEWORK_COLUMN_BYTES,
      Bytes.toBytes(jobDesc.getFramework().toString()));

  // Avoid doing string to byte conversion inside loop.
  byte[] jobConfColumnPrefix =
      Bytes.toBytes(Constants.JOB_CONF_COLUMN_PREFIX + Constants.SEP);

  // Create puts for all the parameters in the job configuration
  Iterator<Entry<String, String>> jobConfIterator = jobConf.iterator();
  while (jobConfIterator.hasNext()) {
    Entry<String, String> entry = jobConfIterator.next();
    // Prefix the job conf entry column with an indicator to
    byte[] column =
        Bytes.add(jobConfColumnPrefix, Bytes.toBytes(entry.getKey()));
    jobPut.addColumn(Constants.INFO_FAM_BYTES, column,
        Bytes.toBytes(entry.getValue()));
  }

  // ensure pool/queuename is set correctly
  setHravenQueueNamePut(jobConf, jobPut, jobKey, jobConfColumnPrefix);

  puts.add(jobPut);

  return puts;
}
 

public void testBug50574() throws Exception {
  Properties props = new Properties();
  int mcastPort = AvailablePort.getRandomAvailablePort(AvailablePort.JGROUPS);
  props.put("mcast-port", String.valueOf(mcastPort));
  Connection conn = TestUtil.getConnection(props);
  Statement st = conn.createStatement();
  
  
  st.execute(" create hdfsstore sensorStore  NameNode 'localhost'  HomeDir './sensorStore' " +
      "BatchTimeInterval 10 milliseconds WriteOnlyFileRolloverInterval 1 seconds");
  
  st.execute(" create table raw_sensor(   id bigint primary key,   timestamp bigint, house_id integer   ) " +
      "partition by column (house_id) persistent hdfsstore (sensorStore) writeonly");
  st.execute("insert into raw_sensor (   id,   timestamp , house_id ) values (1,1,1)");
  st.execute("insert into raw_sensor (   id,   timestamp , house_id ) values (11,11,11)");
  Thread.sleep(2000); 
  
  TestUtil.shutDown();
  deleteOplogs();
  props.put("mcast-port", "0");
  props.put("persist-dd", "false");
  props.put(Property.HADOOP_IS_GFXD_LONER, "true");
  props.put(Property.GFXD_HD_NAMENODEURL, "localhost");
  props.put(Property.GFXD_HD_HOMEDIR, "./sensorStore");
  Configuration conf = new Configuration();
  Iterator<Entry<String, String>> confentries = conf.iterator();
  
  while (confentries.hasNext()){
    Entry<String, String> entry = confentries.next();
    props.put(Property.HADOOP_GFXD_LONER_PROPS_PREFIX + entry.getKey(), entry.getValue());
  }
  
  conn = TestUtil.getConnection(props);
  st = conn.createStatement();
  noOplogsCreatedCheck();
  
  TestUtil.shutDown();
}
 

private static void writeConfInPB(OutputStream dos, Configuration conf) throws IOException {
  ConfigurationProto.Builder confProtoBuilder = ConfigurationProto.newBuilder();
  Iterator<Entry<String, String>> iter = conf.iterator();
  while (iter.hasNext()) {
    Entry<String, String> entry = iter.next();
    PlanKeyValuePair.Builder kvp = PlanKeyValuePair.newBuilder();
    kvp.setKey(entry.getKey());
    kvp.setValue(entry.getValue());
    confProtoBuilder.addConfKeyValues(kvp);
  }
  ConfigurationProto confProto = confProtoBuilder.build();
  confProto.writeTo(dos);
}
 

public SerializableHadoopConfig(@NonNull Configuration configuration){
    this.configuration = configuration;
    content = new LinkedHashMap<>();
    Iterator<Map.Entry<String,String>> iter = configuration.iterator();
    while(iter.hasNext()){
        Map.Entry<String,String> next = iter.next();
        content.put(next.getKey(), next.getValue());
    }
}
 

private static Properties toProperties(Configuration conf)
{
  Iterator<Entry<String, String>> it = conf.iterator();
  Properties props = new Properties();
  while (it.hasNext()) {
    Entry<String, String> e = it.next();
    props.put(e.getKey(), e.getValue());
  }
  return props;
}
 

private Configuration filterByBlacklist(Configuration originalConf, List<String> blacklist) {
    if (blacklist != null && blacklist.size() > 0) {
        Configuration filteredConf = new Configuration(false);
        Iterator<Entry<String, String>> iterator = originalConf.iterator();
        while (iterator.hasNext()) {
            Entry<String, String> next = iterator.next();
            if (blacklist.contains(next.getKey())) {
                continue;
            }
            filteredConf.set(next.getKey(), next.getValue());
        }
        originalConf = filteredConf;
    }
    return originalConf;
}
 

public HConfiguration(Configuration other) {
    if (other != null) {
        Iterator<Map.Entry<String, String>> iter = other.iterator();
        
        while (iter.hasNext()) {
            Map.Entry<String, String> entry = iter.next();
            
            put(entry.getKey(), entry.getValue());
        }
    }
}
 

private static void populateFromMasterConfig(Configuration masterConf, Configuration conf) {
  Iterator<Entry<String, String>> it = masterConf.iterator();
  while (it.hasNext()) {
    Entry<String, String> e = it.next();
    conf.set(e.getKey(), e.getValue());
  }
}
 

protected Map<String, String> getContents(Configuration conf) {
    Map<String, String> contents = new HashMap<String, String>();
    Iterator<Entry<String, String>> iter = conf.iterator();
    while (iter.hasNext()) {
        Entry<String, String> entry = iter.next();

        contents.put(entry.getKey(), entry.getValue());
    }

    return contents;
}
 

/**
 * Constructs a mapping of configuration and includes all properties that start with the specified
 * configuration prefix.  Property names in the mapping are trimmed to remove the configuration prefix.
 * This is a method from Hadoop's Configuration class ported here to make older and custom versions of Hadoop
 * work with JDBC profile.
 *
 * @param configuration configuration map
 * @param confPrefix    configuration prefix
 * @return mapping of configuration properties with prefix stripped
 */
private Map<String, String> getPropsWithPrefix(Configuration configuration, String confPrefix) {
    Map<String, String> configMap = new HashMap<>();
    Iterator<Map.Entry<String, String>> it = configuration.iterator();
    while (it.hasNext()) {
        String propertyName = it.next().getKey();
        if (propertyName.startsWith(confPrefix)) {
            // do not use value from the iterator as it might not come with variable substitution
            String value = configuration.get(propertyName);
            String keyName = propertyName.substring(confPrefix.length());
            configMap.put(keyName, value);
        }
    }
    return configMap;
}
 

/**
  * Pull properties and values from filename.
  *
  * @param filename XML filename
  * @return HashMap containing <Property,Value> entries from XML file
  */
 private HashMap<String,String> extractPropertiesFromXml
     (String filename) {
   if (filename==null) {
     return null;
   }

   // Iterate through XML file for name/value pairs
   Configuration conf = new Configuration(false);
   conf.setAllowNullValueProperties(true);
   conf.addResource(filename);

   HashMap<String,String> retVal = new HashMap<String,String>();
   Iterator<Map.Entry<String,String>> kvItr = conf.iterator();
   while (kvItr.hasNext()) {
     Map.Entry<String,String> entry = kvItr.next();
     String key = entry.getKey();
     // Ignore known xml props
     if (xmlPropsToSkipCompare != null) {
       if (xmlPropsToSkipCompare.contains(key)) {
         continue;
       }
     }
     // Ignore known xml prefixes
     boolean skipPrefix = false;
     if (xmlPrefixToSkipCompare != null) {
       for (String xmlPrefix : xmlPrefixToSkipCompare) {
  if (key.startsWith(xmlPrefix)) {
    skipPrefix = true;
           break;
  }
}
     }
     if (skipPrefix) {
       continue;
     }
     if (conf.onlyKeyExists(key)) {
       retVal.put(key,null);
     } else {
       String value = conf.get(key);
       if (value!=null) {
         retVal.put(key,entry.getValue());
       }
     }
     kvItr.remove();
   }
   return retVal;
 }
 

public void setParameters(Configuration conf) {
  for( Iterator<Entry<String, String>> iter = conf.iterator(); iter.hasNext();) {
    Map.Entry<String, String> entry = iter.next();
    this.hiveConf.set(entry.getKey(), entry.getValue());
  }
}
 

public static Configuration combineConfigurations(Configuration primary, Map<String, Configuration> failovers ) {

    Configuration resultingConfig = new Configuration();
    resultingConfig.clear();

    boolean isFirst = true;
    StringBuilder failOverClusterNames = new StringBuilder();


    Iterator<Entry<String, String>> primaryIt =  primary.iterator();

    while(primaryIt.hasNext()) {
      Entry<String, String> primaryKeyValue = primaryIt.next();
      resultingConfig.set(primaryKeyValue.getKey().replace('_', '.'), primaryKeyValue.getValue());
    }


    for (Entry<String, Configuration> failover: failovers.entrySet()) {
      if (isFirst) {
        isFirst = false;
      } else {
        failOverClusterNames.append(",");
      }
      failOverClusterNames.append(failover.getKey());

      Configuration failureConfig = failover.getValue();

      Iterator<Entry<String, String>> it = failureConfig.iterator();
      while (it.hasNext()) {
        Entry<String, String> keyValue = it.next();

        LOG.info(" -- Looking at : " + keyValue.getKey() + "=" + keyValue.getValue());

        String configKey = keyValue.getKey().replace('_', '.');

        if (configKey.startsWith("hbase.")) {
          resultingConfig.set(ConfigConst.HBASE_FAILOVER_CLUSTER_CONFIG + "." + failover.getKey() + "." + configKey , keyValue.getValue());
          LOG.info(" - Porting config: " + configKey + "=" + keyValue.getValue());
        }
      }
    }

    resultingConfig.set(ConfigConst.HBASE_FAILOVER_CLUSTERS_CONFIG, failOverClusterNames.toString());

    return resultingConfig;
  }
 

/**
 * Put all configuration properties in a given {@link Configuration} object into a given
 * {@link Properties} object.
 *
 * @param configuration the given {@link Configuration} object
 * @param properties the given {@link Properties} object
 */
public static void putConfigurationIntoProperties(Configuration configuration, Properties properties) {
  for (Iterator<Entry<String, String>> it = configuration.iterator(); it.hasNext();) {
    Entry<String, String> entry = it.next();
    properties.put(entry.getKey(), entry.getValue());
  }
}