Java Code Examples for com.typesafe.config.Config#entrySet()

@Override
public <T extends State> StateStore<T> createStateStore(Config config, Class<T> stateClass) {
  // Add all job configuration properties so they are picked up by Hadoop
  Configuration conf = new Configuration();
  for (Map.Entry<String, ConfigValue> entry : config.entrySet()) {
    conf.set(entry.getKey(), entry.getValue().unwrapped().toString());
  }

  try {
    String stateStoreFsUri = ConfigUtils.getString(config, ConfigurationKeys.STATE_STORE_FS_URI_KEY,
        ConfigurationKeys.LOCAL_FS_URI);
    FileSystem stateStoreFs = FileSystem.get(URI.create(stateStoreFsUri), conf);
    String stateStoreRootDir = config.getString(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY);

    return new FsStateStore(stateStoreFs, stateStoreRootDir, stateClass);
  } catch (IOException e) {
    throw new RuntimeException("Failed to create FsStateStore with factory", e);
  }
}
 

public AzkabanCompactionJobLauncher(String jobId, Properties props) {
  super(jobId, LOG);
  this.properties = new Properties();
  this.properties.putAll(props);

  // load dynamic configuration and add them to the job properties
  Config propsAsConfig = ConfigUtils.propertiesToConfig(props);
  DynamicConfigGenerator dynamicConfigGenerator =
      DynamicConfigGeneratorFactory.createDynamicConfigGenerator(propsAsConfig);
  Config dynamicConfig = dynamicConfigGenerator.generateDynamicConfig(propsAsConfig);

  // add the dynamic config to the job config
  for (Map.Entry<String, ConfigValue> entry : dynamicConfig.entrySet()) {
    this.properties.put(entry.getKey(), entry.getValue().unwrapped().toString());
  }

  this.compactor = getCompactor(getCompactorFactory(), getCompactorListener(getCompactorListenerFactory()));
}
 

/**
 * Resolves encrypted config value(s) by considering on the path with "encConfigPath" as encrypted.
 * (If encConfigPath is absent or encConfigPath does not exist in config, config will be just returned untouched.)
 * It will use Password manager via given config. Thus, convention of PasswordManager need to be followed in order to be decrypted.
 * Note that "encConfigPath" path will be removed from the config key, leaving child path on the config key.
 * e.g:
 *  encConfigPath = enc.conf
 *  - Before : { enc.conf.secret_key : ENC(rOF43721f0pZqAXg#63a) }
 *  - After  : { secret_key : decrypted_val }
 *
 * @param config
 * @param encConfigPath
 * @return
 */
public static Config resolveEncrypted(Config config, Optional<String> encConfigPath) {
  if (!encConfigPath.isPresent() || !config.hasPath(encConfigPath.get())) {
    return config;
  }

  Config encryptedConfig = config.getConfig(encConfigPath.get());

  PasswordManager passwordManager = PasswordManager.getInstance(configToProperties(config));
  Map<String, String> tmpMap = Maps.newHashMap();
  for (Map.Entry<String, ConfigValue> entry : encryptedConfig.entrySet()) {
    String val = entry.getValue().unwrapped().toString();
    val = passwordManager.readPassword(val);
    tmpMap.put(entry.getKey(), val);
  }
  return ConfigFactory.parseMap(tmpMap).withFallback(config);
}
 

@Override
public Config apply(final String systemVcapServices) {
    checkNotNull(systemVcapServices, "system VCAP services string");
    if (systemVcapServices.isEmpty()) {
        return ConfigFactory.empty();
    }

    final Config vcapServicesConfig = tryToParseString(systemVcapServices);
    final Set<Map.Entry<String, ConfigValue>> vcapServicesConfigEntries = vcapServicesConfig.entrySet();

    final Map<String, Object> result = new HashMap<>(vcapServicesConfigEntries.size());
    for (final Map.Entry<String, ConfigValue> serviceConfigEntry : vcapServicesConfigEntries) {
        result.put(serviceConfigEntry.getKey(), convertConfigListToConfigObject(serviceConfigEntry.getValue()));
    }
    return ConfigFactory.parseMap(result);
}
 

private static Config applySystemProperties(Config config, Config reference){
  for (Entry<String, ConfigValue> entry : reference.entrySet()) {
    String property = System.getProperty(entry.getKey());
    if (property != null && !property.isEmpty()) {
      // hack to deal with array of strings
      if (property.startsWith("[") && property.endsWith("]")) {
        property = property.substring(1, property.length()-1);
        if (property.trim().isEmpty()) {
          continue;
        }
        String[] strings = property.split(",");
        if (strings != null && strings.length > 0) {
          List<String> listStrings = new ArrayList<>();
          for (String str : strings) {
            listStrings.add(str.trim());
          }
          config = config.withValue(entry.getKey(), ConfigValueFactory.fromAnyRef(listStrings));
        }
      } else {
        config = config.withValue(entry.getKey(), ConfigValueFactory.fromAnyRef(property));
      }
      logger.info("Applying provided system property to config: -D{}={}", entry.getKey(), property);
    }
  }
  return config;
}
 

void parseJson(String jsonString, LogRecord logRecord) {
  String cleanedJsonString = IngestUtilities.removeUnprintableCharacters(jsonString);
  Config cfg = ConfigFactory.parseString(cleanedJsonString);
  for (Map.Entry<String, ConfigValue> entry : cfg.entrySet()) {
    String key = entry.getKey();
    ConfigValue value = entry.getValue();
    switch (value.valueType()) {
      case BOOLEAN:
      case NUMBER:
      case OBJECT:
      case STRING:
        logRecord.setValue(key, ObjectUtils.toString(value.unwrapped()));
        break;
      case LIST:
        ConfigList list = (ConfigList) value;
        Gson gson = new Gson();
        String json = gson.toJson(list.unwrapped());
        logRecord.setValue(key, json);
        break;
      case NULL:
      default:
        // skip
    }
  }
}
 

@Override
public void configure(Config config) {
  tableName = config.getString(TABLE_CONFIG);

  if (config.hasPath(PARTITION_BY_CONFIG)) {
    List<String> colNames = config.getStringList(PARTITION_BY_CONFIG);
    partitionColumns = colNames.toArray(new String[0]);
  }

  doesAlignColumns = ConfigUtils.getOrElse(config, ALIGN_COLUMNS_CONFIG, false);

  if (config.hasPath(LOCATION_CONFIG) || config.hasPath(OPTIONS_CONFIG)) {
    options = new ConfigUtils.OptionMap(config);

    if (config.hasPath(LOCATION_CONFIG)) {
      options.resolve("path", LOCATION_CONFIG);
    }

    if (config.hasPath(OPTIONS_CONFIG)) {
      Config optionsConfig = config.getConfig(OPTIONS_CONFIG);
      for (Map.Entry<String, ConfigValue> entry : optionsConfig.entrySet()) {
        String param = entry.getKey();
        String value = entry.getValue().unwrapped().toString();
        if (value != null) {
          options.put(param, value);
        }
      }
    }
  }
}
 

private static Map<String, Object> toMap(Config config) {
  Map<String, Object> map = new HashMap<>();
  for (Map.Entry<String, ConfigValue> entry: config.entrySet()) {
    map.put(entry.getKey(), entry.getValue().unwrapped());
  }
  return map;
}
 

/**
 * Check that every key-value in superConfig is in subConfig
 */
public static boolean verifySubset(Config superConfig, Config subConfig) {
  for (Map.Entry<String, ConfigValue> entry : subConfig.entrySet()) {
    if (!superConfig.hasPath(entry.getKey()) || !superConfig.getValue(entry.getKey()).unwrapped()
        .equals(entry.getValue().unwrapped())) {
      return false;
    }
  }
  return true;
}
 

private static SparkNameFunction fromPatternList(List<? extends Config> patterns, Registry registry) {
  final List<NameMatcher> matchers = new ArrayList<>();
  for (Config config : patterns) {
    final Pattern pattern = Pattern.compile(config.getString("pattern"));
    final Map<String, Integer> tagsMap = new LinkedHashMap<>();
    final Config tagsCfg = config.getConfig("tags");
    for (Map.Entry<String, ConfigValue> entry : tagsCfg.entrySet()) {
      tagsMap.put(entry.getKey(), (Integer) entry.getValue().unwrapped());
    }
    matchers.add(new NameMatcher(pattern, registry, config.getInt("name"), tagsMap));
  }
  return new SparkNameFunction(matchers);
}
 

/**
 * Creates the Property Builder based on reference.conf files
 * @throws Exception exception generated while creating source
 */
private void writeDaoBuilder() throws Exception {
    String className = "com.tmobile.opensource.casquatch.CasquatchDaoBuilder";
    Map<String, Object> input = inputStart(className);

    ConfigFactory.invalidateCaches();
    Config root = ConfigFactory.load(this.getClass().getClassLoader(),"reference.conf");
    Config config = root.getConfig("casquatch-template").withFallback(root.getConfig("casquatch-defaults").withFallback(root.getConfig("datastax-java-driver")));
    Map<String,String> properties = new HashMap<>();
    for(Map.Entry entry : config.entrySet()) {
        properties.put(entry.getKey().toString(),((ConfigValue)entry.getValue()).valueType().name());
    }
    input.put("properties", properties);
    createSource(className,"CasquatchDaoBuilder.ftl",input);
}
 

public Set<Map.Entry<String, Object>> getEntrySet(Config config) {
  Map<String, Object> map = Maps.newHashMap();
  for (Map.Entry<String, ConfigValue> entry : config.entrySet()) {
    map.put(trimQuote(entry.getKey()), entry.getValue().unwrapped());
  }
  return map.entrySet();
}
 

public static Configuration getHBaseConfiguration(Config config) throws IOException {
  Configuration hbaseConfiguration = HBaseConfiguration.create();
  if (config.hasPath(ZK_QUORUM_PROPERTY)) {
    String zkQuorum = config.getString(ZK_QUORUM_PROPERTY);
    hbaseConfiguration.set(HConstants.ZOOKEEPER_QUORUM, zkQuorum);
  }
  LOG.debug("HBase:: Using ZK quorum: {}", hbaseConfiguration.get(HConstants.ZOOKEEPER_QUORUM));
  LOG.debug("HBase:: Using security: {}", hbaseConfiguration.get("hadoop.security.authentication"));

  // Add any other pass-through options starting with HBASE_PASSTHRU_PREFIX
  if (config.hasPath(HBASE_PASSTHRU_PREFIX)) {
    Config hbaseConfigs = config.getConfig(HBASE_PASSTHRU_PREFIX);
    for (Map.Entry<String, ConfigValue> entry : hbaseConfigs.entrySet()) {
      String param = entry.getKey();
      String value = null;
      switch (entry.getValue().valueType()) {
        case STRING:
          value = (String) entry.getValue().unwrapped();
          break;
        default:
          LOG.warn("Only string parameters currently " +
              "supported, auto-converting to String [{}]", param);
          value = entry.getValue().unwrapped().toString();
      }
      if (value != null) {
        hbaseConfiguration.set(param, value);
      }
    }
  }
  return hbaseConfiguration;
}
 

private void check(){
  final Config inner = getInnerConfig();
  final Config ref = reference.resolve();

  // make sure types are right
  inner.checkValid(ref);

  // make sure we don't have any extra paths. these are typically typos.
  List<String> invalidPaths = new ArrayList<>();
  for(Entry<String, ConfigValue> entry : inner.entrySet()){
    if(!ref.hasPath(entry.getKey())){
      invalidPaths.add(entry.getKey());
    }
  }

  if(!invalidPaths.isEmpty()){
    StringBuilder sb = new StringBuilder();
    sb.append("Failure reading configuration file. The following properties were invalid:\n");
    for(String s : invalidPaths){
      sb.append("\t");
      sb.append(s);
      sb.append("\n");
    }

    throw new RuntimeException(sb.toString());
  }
}
 

/**
 * Get configuration for name. Returns empty if missing and logs values to trace
 * @param configName name of configuration
 * @return Config object
 */
private static Config getConfig(String configName) {
    if(root().hasPath(configName)) {
        log.debug("Loading {}",configName);
        Config newConfig = root().getConfig(configName);
        for (Map.Entry<String, ConfigValue> entry : newConfig.entrySet()) {
            log.debug("{}: {} -> {}",configName,entry.getKey(),entry.getValue().render());
        }
        return newConfig;
    }
    else {
        log.trace("No values found for {}",configName);
        return ConfigFactory.empty();
    }
}
 

static void addOverrides(Configuration conf, Config config) {
  for(Map.Entry<String, ConfigValue> entry: config.entrySet()) {
    String propName = entry.getKey();
    if (propName.endsWith(STRIP_SUFFIX)) {
      propName = propName.substring(0, propName.length() - STRIP_SUFFIX.length());
    }
    conf.set(propName, entry.getValue().unwrapped().toString());
  }
}
 

private void assertNameValueEquals2(String key, String value, Config config) {
  for (Map.Entry<String, ConfigValue> entry : config.entrySet()) {
    //assertEquals(key, entry.getKey()); // would fail
    assertEquals(key, trimQuote(entry.getKey()));
    String actualValue = entry.getValue().unwrapped().toString();
    assertEquals(value, actualValue);
    actualValue = config.getString(entry.getKey());
    assertEquals(value, actualValue);
  }
}
 

public void printConfig(Config config){
  Set<Map.Entry<String,ConfigValue>> entrySet = config.entrySet();
  for(Map.Entry<String,ConfigValue> entry: entrySet){
    System.out.println("key: " + entry.getKey() + ", value: " + entry.getValue());
  }
}
 

public void printConfig(Config config){
  Set<Map.Entry<String,ConfigValue>> entrySet = config.entrySet();
  for(Map.Entry<String,ConfigValue> entry: entrySet){
    System.out.println("key: " + entry.getKey() + ", value: " + entry.getValue());
  }
}
 

/**
 * Provides Hadoop configuration given state.
 * It also supports decrypting values on "encryptedPath".
 * Note that this encryptedPath path will be removed from full path of each config key and leaving only child path on the key(s).
 * If there's same config path as child path, the one stripped will have higher priority.
 *
 * e.g:
 * - encryptedPath: writer.fs.encrypted
 *   before: writer.fs.encrypted.secret
 *   after: secret
 *
 * Common use case for these encryptedPath:
 *   When there's have encrypted credential in job property but you'd like Filesystem to get decrypted value.
 *
 * @param srcConfig source config.
 * @param encryptedPath Optional. If provided, config that is on this path will be decrypted. @see ConfigUtils.resolveEncrypted
 *                      Note that config on encryptedPath will be included in the end result even it's not part of includeOnlyPath
 * @return Hadoop Configuration.
 */
public static Configuration getConfFromState(State state, Optional<String> encryptedPath) {
  Config config = ConfigFactory.parseProperties(state.getProperties());
  if (encryptedPath.isPresent()) {
    config = ConfigUtils.resolveEncrypted(config, encryptedPath);
  }
  Configuration conf = newConfiguration();

  for (Entry<String, ConfigValue> entry : config.entrySet()) {
    conf.set(entry.getKey(), entry.getValue().unwrapped().toString());
  }
  return conf;
}