Java Code Examples for com.typesafe.config.Config#withFallback()

public EncryptionConfig(Config encryptionConfig) throws IOException {
  this.encryptionAlgorithm = ConfigUtils.getString(encryptionConfig, DatasetDescriptorConfigKeys.ENCRYPTION_ALGORITHM_KEY,
      DatasetDescriptorConfigKeys.DATASET_DESCRIPTOR_CONFIG_ANY).toLowerCase();
  if (this.encryptionAlgorithm.equalsIgnoreCase(DatasetDescriptorConfigKeys.DATASET_DESCRIPTOR_CONFIG_NONE)) {
    this.keystoreType = DatasetDescriptorConfigKeys.DATASET_DESCRIPTOR_CONFIG_NONE;
    this.keystoreEncoding = DatasetDescriptorConfigKeys.DATASET_DESCRIPTOR_CONFIG_NONE;
    this.encryptionLevel = DatasetDescriptorConfigKeys.DATASET_DESCRIPTOR_CONFIG_NONE;
    this.encryptedFields = DatasetDescriptorConfigKeys.DATASET_DESCRIPTOR_CONFIG_NONE;
  } else {
    this.keystoreType = ConfigUtils.getString(encryptionConfig, DatasetDescriptorConfigKeys.ENCRYPTION_KEYSTORE_TYPE_KEY,
        DatasetDescriptorConfigKeys.DATASET_DESCRIPTOR_CONFIG_ANY).toLowerCase();
    this.keystoreEncoding = ConfigUtils.getString(encryptionConfig, DatasetDescriptorConfigKeys.ENCRYPTION_KEYSTORE_ENCODING_KEY,
        DatasetDescriptorConfigKeys.DATASET_DESCRIPTOR_CONFIG_ANY).toLowerCase();
    this.encryptionLevel = ConfigUtils.getString(encryptionConfig, DatasetDescriptorConfigKeys.ENCRYPTION_LEVEL_KEY,
        DatasetDescriptorConfigKeys.DATASET_DESCRIPTOR_CONFIG_ANY).toLowerCase();
    this.encryptedFields = ConfigUtils.getString(encryptionConfig, DatasetDescriptorConfigKeys.ENCRYPTED_FIELDS,
        DatasetDescriptorConfigKeys.DATASET_DESCRIPTOR_CONFIG_ANY).toLowerCase();
    validate(this.encryptionLevel, this.encryptedFields);
  }
  this.rawConfig = encryptionConfig.withFallback(DEFAULT_FALLBACK);
}
 

protected void loadConfig() {
  final Config defaultConfig =
      ConfigFactory.parseReader(
          new InputStreamReader(
              BungeeChat.getInstance().getResourceAsStream(CONFIG_FILE_NAME),
              StandardCharsets.UTF_8),
          PARSE_OPTIONS);
  final Config strippedDefautConfig = defaultConfig.withoutPath("ServerAlias");

  if (CONFIG_FILE.exists()) {
    try {
      Config fileConfig = ConfigFactory.parseFile(CONFIG_FILE, PARSE_OPTIONS);

      config = fileConfig.withFallback(strippedDefautConfig);
    } catch (ConfigException e) {
      LoggerHelper.error("Error while reading config:\n" + e.getLocalizedMessage());

      config = defaultConfig;
    }
  } else {
    config = defaultConfig;
  }

  config = config.resolve();

  convertOldConfig();
  // Reapply default config. By default this does nothing but it can fix the missing config
  // settings in some cases
  config = config.withFallback(strippedDefautConfig);
  copyComments(defaultConfig);

  saveConfig();
}
 

@Override
protected Config getLocallyResolvedConfig(Config userConfig) throws TemplateException {
  for (String required : this.required) {
    if (!userConfig.hasPath(required)) {
      throw new TemplateException("Missing required property " + required);
    }
  }
  return userConfig.withFallback(getLocalRawTemplate());
}
 

@Override
protected Collection<Config> getTaskDriverConfigs() {
  // task driver config initialization
  URL url = Resources.getResource("BasicTaskDriver.conf");
  Config taskDriverConfig = ConfigFactory.parseURL(url);
  taskDriverConfig = taskDriverConfig.withFallback(getClusterConfig());
  Config taskDriver1 = addInstanceName(taskDriverConfig, "TaskDriver1");
  return ImmutableList.of(taskDriver1);
}
 

private Config getJobConfig() throws IOException {
  try (InputStream resourceStream = Resources.getResource(JOB_CONF_NAME).openStream()) {
    Reader reader = new InputStreamReader(resourceStream);
    Config rawJobConfig =
        ConfigFactory.parseReader(reader, ConfigParseOptions.defaults().setSyntax(ConfigSyntax.CONF));
    rawJobConfig = rawJobConfig.withFallback(getClusterConfig());

    Config newConfig = ClusterIntegrationTestUtils.buildSleepingJob(JOB_ID, TASK_STATE_FILE, 100L);

    newConfig = newConfig.withValue(SleepingTask.TASK_STATE_FILE_KEY, ConfigValueFactory.fromAnyRef(TASK_STATE_FILE));
    newConfig = newConfig.withFallback(rawJobConfig);
    return newConfig;
  }
}
 

/**
 * Returns a new instance of {@code ConfigWithFallback} based on the given arguments.
 *
 * @param originalConfig the original Config which is supposed to provide a nested Config at {@code configPath} and
 * which will be extended by the fall back config based on {@code fallBackValues}.
 * @param fallBackValues base for the fall back which is applied to the original Config within
 * {@code originalConfig} at {@code configPath}.
 * @return the instance.
 * @throws DittoConfigError if any argument is {@code null} or if the value of {@code originalConfig} at
 * {@code configPath} is not of type {@link com.typesafe.config.ConfigValueType#OBJECT}.
 */
public static ConfigWithFallback newInstance(final Config originalConfig, final KnownConfigValue[] fallBackValues) {

    validateArgument(originalConfig, "original Config");
    validateArgument(fallBackValues, "fall-back values");

    Config baseConfig = originalConfig;
    if (0 < fallBackValues.length) {
        baseConfig = baseConfig.withFallback(arrayToConfig(fallBackValues));
    }

    return new ConfigWithFallback(baseConfig, "");
}
 

private Config getResolvedConfigRecursive(ConfigKeyPath configKey, Set<String> alreadyLoadedPaths,
    Optional<Config> runtimeConfig) {

  if (this.cs instanceof ConfigStoreWithResolution) {
    return ((ConfigStoreWithResolution) this.cs).getResolvedConfig(configKey, this.version);
  }

  if (!alreadyLoadedPaths.add(configKey.getAbsolutePathString())) {
    return ConfigFactory.empty();
  }

  Config initialConfig = this.getOwnConfig(configKey);
  if (configKey.isRootPath()) {
    return initialConfig;
  }

  List<ConfigKeyPath> ownImports = this.topology.getOwnImports(configKey, runtimeConfig);
  // merge with other configs from imports
  if (ownImports != null) {
    for (ConfigKeyPath p : ownImports) {
      initialConfig =
          initialConfig.withFallback(this.getResolvedConfigRecursive(p, alreadyLoadedPaths, runtimeConfig));
    }
  }

  // merge with configs from parent for Non root
  initialConfig = initialConfig
      .withFallback(this.getResolvedConfigRecursive(configKey.getParent(), alreadyLoadedPaths, runtimeConfig));

  return initialConfig;
}
 

/**
 * Return the root configuration loaded from the helios configuration files.
 */
static Config loadConfig() {
  final ConfigResolveOptions resolveOptions = ConfigResolveOptions
      .defaults()
      .setAllowUnresolved(true);

  final ConfigParseOptions parseOptions = ConfigParseOptions.defaults();

  final Config baseConfig = ConfigFactory.load(BASE_CONFIG_FILE, parseOptions, resolveOptions);

  final Config appConfig = ConfigFactory.load(APP_CONFIG_FILE, parseOptions, resolveOptions);

  return appConfig.withFallback(baseConfig);
}
 

public TimeBasedDatasetStoreDataset(Key key, List<DatasetStateStoreEntryManager> entries, Properties props) {
  super(key, entries);
  this.versionFinder = new TimestampedDatasetStateStoreVersionFinder();
  Config propsAsConfig = ConfigUtils.propertiesToConfig(props);

  // strip the retention config namespace since the selection policy looks for configuration without the namespace
  Config retentionConfig = ConfigUtils.getConfigOrEmpty(propsAsConfig,
      ConfigurableCleanableDataset.RETENTION_CONFIGURATION_KEY);
  Config retentionConfigWithFallback = retentionConfig.withFallback(propsAsConfig);

  this.versionSelectionPolicy = createSelectionPolicy(ConfigUtils.getString(retentionConfigWithFallback,
      SELECTION_POLICY_CLASS_KEY, DEFAULT_SELECTION_POLICY_CLASS), retentionConfigWithFallback, props);
}
 

/**
 * @see <a href="https://github.com/ben-manes/caffeine/blob/master/simulator/src/main/resources/reference.conf"/>
 */
@Override
protected Policy createCaffeinePolicy(final Config configWithSize) {
	Tuning tuning = getTuning();
	String config = "lirs {\n" +
		"  # The percentage for the HOT queue\n" +
		"  percent-hot = \"" + tuning.ratioHot + "\"\n" +
		"  # The multiple of the maximum size dedicated to non-resident entries\n" +
		"  non-resident-multiplier = \"2.0\"\n" +
		"  # The percentage of the hottest entries where the stack move is skipped\n" +
		"  percent-fast-path = \"" +tuning.ratioFastPath +"\" # \"0.05\" is reasonable\n" +
		"}\n";
   Config f = ConfigFactory.parseString(config);
   return new LirsPolicy(f.withFallback(configWithSize));
}
 

public void testSerDerWithConfig() throws IOException {
  ServiceRequester sr1 = new ServiceRequester("kafkaetl", "user", "dv");
  ServiceRequester sr2 = new ServiceRequester("gobblin", "group", "dv");
  ServiceRequester sr3 = new ServiceRequester("crm-backend", "service", "cert");

  List<ServiceRequester> list = new ArrayList<>();
  sr1.getProperties().put("customKey", "${123}");
  list.add(sr1);
  list.add(sr2);
  list.add(sr3);

  String serialize = RequesterService.serialize(list);
  Properties props = new Properties();

  props.put(RequesterService.REQUESTER_LIST, serialize);

  // config creation must happen this way because in FlowConfigResourceLocalHandler we read the flowconfig like this
  Config initConfig = ConfigBuilder.create().build();
  Config config = initConfig.withFallback(ConfigFactory.parseString(props.toString()).resolve());

  Properties props2 = ConfigUtils.configToProperties(config);
  String serialize2 = props2.getProperty(RequesterService.REQUESTER_LIST);

  // This may not hold true unless we use/write a json comparator
  // Assert.assertEquals(serialize2, serialize);
  List<ServiceRequester> list2 = RequesterService.deserialize(serialize);
  Assert.assertEquals(list2, list);
}
 

public KafkaJobStatusMonitor(String topic, Config config, int numThreads)
    throws ReflectiveOperationException {
  super(topic, config.withFallback(DEFAULTS), numThreads);
  String stateStoreFactoryClass = ConfigUtils.getString(config, ConfigurationKeys.STATE_STORE_FACTORY_CLASS_KEY, FileContextBasedFsStateStoreFactory.class.getName());

  this.stateStore =
      ((StateStore.Factory) Class.forName(stateStoreFactoryClass).newInstance()).createStateStore(config, org.apache.gobblin.configuration.State.class);
  this.scheduledExecutorService = Executors.newScheduledThreadPool(1);
}
 

private Config getRawTemplateConfigHelper(Set<JobTemplate> alreadyInheritedTemplates)
    throws SpecNotFoundException, TemplateException {
  Config rawTemplate = getLocalRawTemplate();
  if (this.superTemplates != null) {
    for (JobTemplate template : Lists.reverse(this.superTemplates)) {
      if (!alreadyInheritedTemplates.contains(template)) {
        alreadyInheritedTemplates.add(template);
        Config thisFallback = template instanceof InheritingJobTemplate ? ((InheritingJobTemplate) template).getRawTemplateConfigHelper(alreadyInheritedTemplates)
            : template.getRawTemplateConfig();
        rawTemplate = rawTemplate.withFallback(thisFallback);
      }
    }
  }
  return rawTemplate;
}
 

@Override
public R2RestWriterBuilder fromConfig(Config config) {
  config = config.withFallback(FALLBACK);
  this.client = createClient(config);

  String urlTemplate = config.getString(HttpConstants.URL_TEMPLATE);
  String verb = config.getString(HttpConstants.VERB);
  String protocolVersion = config.getString(HttpConstants.PROTOCOL_VERSION);
  asyncRequestBuilder = new R2RestRequestBuilder(urlTemplate, verb, protocolVersion);

  Set<String> errorCodeWhitelist = HttpUtils.getErrorCodeWhitelist(config);
  responseHandler = new R2RestResponseHandler(errorCodeWhitelist, metricContext);
  return this;
}
 

/**
 * @see <a href="https://github.com/ben-manes/caffeine/blob/master/simulator/src/main/resources/reference.conf"/>
 */
@Override
protected Policy createCaffeinePolicy(final Config configWithSize) {
	Tuning tuning = getTuning();
	String config =
		"  # The seed for randomized operations\n" +
		"  random-seed = \"1033096058\"" +
		"\n" +
		"  tiny-lfu {\n" +
		"    # CountMinSketch: count-min-4 (4-bit), count-min-64 (64-bit)\n" +
		"    # Table: random-table, tiny-table, perfect-table\n" +
		"    sketch = \"count-min-4\"\n" +
		"\n" +
		"    # If increments are conservative by only updating the minimum counters for CountMin sketches\n" +
		"    count-min.conservative = false\n" +
		"\n" +
		"    count-min-64 {\n" +
		"      eps = \"0.0001\"\n" +
		"      confidence = \"0.99\"\n" +
		"    }\n" +
		"\n" +
		"    count-min-4 {\n" +
		"      # periodic: Resets by periodically halving all counters\n" +
		"      # incremental: Resets by halving counters in an incremental sweep\n" +
		"      reset = \"periodic\"\n" +
		"\n" +
		"      # The incremental reset interval (the number of additions before halving counters)\n" +
		"      increment = 16\n" +
		"    }\n" +
		"  }\n" +
		" window-tiny-lfu {\n" +
		"    # The percentage for the MAIN space (PROBATION + PROTECTED)\n" +
		"    percent-main = [\"" + tuning.percentMain + "\"]\n" +
		"    # The percentage for the PROTECTED MAIN queue\n" +
		"    percent-main-protected = \"0.80\"\n" +
		"    # The percentage of the hottest entries where the PROTECTED move is skipped\n" +
		"    percent-fast-path = \"0.0\" # \"0.05\" is reasonable\n" +
		"  }\n";
   Config f = ConfigFactory.parseString(config);
   return new LirsPolicy(f.withFallback(configWithSize));
}
 

@Test
public void testContains() throws IOException {
  //Ensure descriptor2's path is matched by the regular expression in descriptor1's path
  Config config1 = ConfigFactory.empty().withValue(DatasetDescriptorConfigKeys.PATH_KEY, ConfigValueFactory.fromAnyRef("/a/b/c/*"))
      .withValue(DatasetDescriptorConfigKeys.PLATFORM_KEY, ConfigValueFactory.fromAnyRef("hdfs"));
  FSDatasetDescriptor descriptor1 = new FSDatasetDescriptor(config1);

  Config config2 = ConfigFactory.empty().withValue(DatasetDescriptorConfigKeys.PATH_KEY, ConfigValueFactory.fromAnyRef("/a/b/c/d"))
      .withValue(DatasetDescriptorConfigKeys.PLATFORM_KEY, ConfigValueFactory.fromAnyRef("hdfs"))
      .withValue(DatasetDescriptorConfigKeys.FORMAT_KEY, ConfigValueFactory.fromAnyRef("avro"))
      .withValue(DatasetDescriptorConfigKeys.CODEC_KEY, ConfigValueFactory.fromAnyRef("gzip"));

  FSDatasetDescriptor descriptor2 = new FSDatasetDescriptor(config2);
  Assert.assertTrue(descriptor1.contains(descriptor2));

  //Add encryption config
  Config encConfig = ConfigFactory.empty().withValue(DatasetDescriptorConfigKeys.ENCRYPTION_LEVEL_KEY, ConfigValueFactory.fromAnyRef("file"))
      .withValue(DatasetDescriptorConfigKeys.ENCRYPTION_ALGORITHM_KEY, ConfigValueFactory.fromAnyRef("aes_rotating"))
      .atPath(DatasetDescriptorConfigKeys.ENCYPTION_PREFIX);
  Config config3 = config2.withFallback(encConfig);
  FSDatasetDescriptor descriptor3 = new FSDatasetDescriptor(config3);
  Assert.assertTrue(descriptor2.contains(descriptor3));
  Assert.assertTrue(descriptor1.contains(descriptor3));

  //Add partition config
  Config partitionConfig = ConfigFactory.empty().withValue(DatasetDescriptorConfigKeys.PARTITION_TYPE_KEY, ConfigValueFactory.fromAnyRef("datetime"))
      .withValue(DatasetDescriptorConfigKeys.PARTITION_PATTERN_KEY, ConfigValueFactory.fromAnyRef("yyyy/MM/dd"))
      .atPath(DatasetDescriptorConfigKeys.PARTITION_PREFIX);
  Config config4 = config3.withFallback(partitionConfig);
  FSDatasetDescriptor descriptor4 = new FSDatasetDescriptor(config4);
  Assert.assertTrue(descriptor3.contains(descriptor4));
  Assert.assertTrue(descriptor2.contains(descriptor4));
  Assert.assertTrue(descriptor1.contains(descriptor4));

  //Add compaction/retention config
  Config miscConfig = ConfigFactory.empty().withValue(DatasetDescriptorConfigKeys.IS_COMPACTED_AND_DEDUPED_KEY, ConfigValueFactory.fromAnyRef("true"))
      .withValue(DatasetDescriptorConfigKeys.IS_RETENTION_APPLIED_KEY, ConfigValueFactory.fromAnyRef("true"));
  Config config5 = config4.withFallback(miscConfig);
  FSDatasetDescriptor descriptor5 = new FSDatasetDescriptor(config5);
  Assert.assertFalse(descriptor4.contains(descriptor5));
  Assert.assertFalse(descriptor3.contains(descriptor5));
  Assert.assertFalse(descriptor2.contains(descriptor5));
  Assert.assertFalse(descriptor1.contains(descriptor5));

  // Test subpaths
  Config subPathConfig = ConfigFactory.empty().withValue(DatasetDescriptorConfigKeys.PATH_KEY, ConfigValueFactory.fromAnyRef("/a/b/c"))
      .withValue(DatasetDescriptorConfigKeys.SUBPATHS_KEY, ConfigValueFactory.fromAnyRef("{e,f,g}"))
      .withValue(DatasetDescriptorConfigKeys.PLATFORM_KEY, ConfigValueFactory.fromAnyRef("hdfs"));
  FSDatasetDescriptor descriptor6 = new FSDatasetDescriptor(subPathConfig);
  Assert.assertTrue(descriptor1.contains(descriptor6));
  Assert.assertFalse(descriptor2.contains(descriptor6));
}
 

public static void addConfig(Config newConfig) {
  config = newConfig.withFallback(config);
}
 

/**
 *
 * @param flowTemplateDirURI Relative URI of the flow template directory
 * @return a list of {@link JobTemplate}s for a given flow identified by its {@link URI}.
 * @throws IOException
 * @throws SpecNotFoundException
 * @throws JobTemplate.TemplateException
 */
public List<JobTemplate> getJobTemplatesForFlow(URI flowTemplateDirURI)
    throws IOException, SpecNotFoundException, JobTemplate.TemplateException, URISyntaxException {

  PathFilter extensionFilter = file -> {
    for (String extension : JOB_FILE_EXTENSIONS) {
      if (file.getName().endsWith(extension)) {
        return true;
      }
    }
    return false;
  };

  if (!validateTemplateURI(flowTemplateDirURI)) {
    throw new JobTemplate.TemplateException(String.format("The FlowTemplate %s is not valid", flowTemplateDirURI));
  }

  List<JobTemplate> jobTemplates = new ArrayList<>();

  String templateCatalogDir = this.sysConfig.getString(ServiceConfigKeys.TEMPLATE_CATALOGS_FULLY_QUALIFIED_PATH_KEY);

  //Flow templates are located under templateCatalogDir/flowEdgeTemplates
  Path flowTemplateDirPath = PathUtils.mergePaths(new Path(templateCatalogDir), new Path(flowTemplateDirURI));
  //Job files (with extension .job) are located under templateCatalogDir/flowEdgeTemplates/jobs directory.
  Path jobFilePath = new Path(flowTemplateDirPath, JOBS_DIR_NAME);

  FileSystem fs = FileSystem.get(jobFilePath.toUri(), new Configuration());

  for (FileStatus fileStatus : fs.listStatus(jobFilePath, extensionFilter)) {
    Config jobConfig = loadHoconFileAtPath(fileStatus.getPath());
    //Check if the .job file has an underlying job template
    if (jobConfig.hasPath(GOBBLIN_JOB_TEMPLATE_KEY)) {
      URI jobTemplateRelativeUri = new URI(jobConfig.getString(GOBBLIN_JOB_TEMPLATE_KEY));
      if (!jobTemplateRelativeUri.getScheme().equals(FS_SCHEME)) {
        throw new RuntimeException(
            "Expected scheme " + FS_SCHEME + " got unsupported scheme " + flowTemplateDirURI.getScheme());
      }
      Path fullJobTemplatePath = PathUtils.mergePaths(new Path(templateCatalogDir), new Path(jobTemplateRelativeUri));
      jobConfig = jobConfig.withFallback(loadHoconFileAtPath(fullJobTemplatePath));
    }
    jobTemplates.add(new HOCONInputStreamJobTemplate(jobConfig, fileStatus.getPath().toUri(), this));
  }
  return jobTemplates;
}
 

@Test
public void appendAliasesWorksAsExpected() {
    final JsonObject originalConfigJson = JsonObject.newBuilder()
            .set("vcap", JsonObject.newBuilder()
                    .set("MongoDB-Service", JsonObject.newBuilder()
                            .set("ditto-mongodb-staging", JsonObject.newBuilder()
                                    .set("binding_name", JsonValue.nullLiteral())
                                    .set("credentials", JsonObject.newBuilder()
                                            .set("readonly", false)
                                            .set("replicaset", "stretched-0815")
                                            .build())
                                    .set("instance_name", "ditto-mongodb-staging")
                                    .set("label", "MongoDB-Service")
                                    .set("name", "ditto-mongodb-staging")
                                    .set("plan", "Database on Dedicated Replica Set - Stretched")
                                    .set("provider", JsonValue.nullLiteral())
                                    .set("syslog_drain_url", JsonValue.nullLiteral())
                                    .set("tags", JsonArray.newBuilder()
                                            .add("mongodb", "mongo", "database", "db", "mongoose")
                                            .build())
                                    .set("volume_mounts", JsonArray.empty())
                                    .build())
                            .build())
                    .build())
            .build();

    final Config vcapConfig = ConfigFactory.parseString(originalConfigJson.toString());

    final JsonObject aliasedConfigJson = JsonObject.newBuilder()
            .set("vcap", JsonObject.newBuilder()
                    .set("mongodb", originalConfigJson.getValue("/vcap/MongoDB-Service/ditto-mongodb-staging")
                            .orElseGet(JsonValue::nullLiteral))
                    .build())
            .build();
    final Config expected = vcapConfig.withFallback(ConfigFactory.parseString(aliasedConfigJson.toString()));

    final AliasesAppender underTest = AliasesAppender.getInstance();
    final Config actual = underTest.apply(vcapConfig);

    assertThat(actual).isEqualTo(expected);
}
 

/**
 * Puts a new config atop of existing stack making the options
 * in the supplied config overriding existing options
 * Once put this config can't be removed
 *
 * @param overrideOptions - atop config
 */
public void overrideParams(Config overrideOptions) {
    config = overrideOptions.withFallback(config);
    validateConfig();
}