Java Code Examples for org.apache.commons.configuration.AbstractConfiguration#containsKey()

public static LoggingAuditClientConfig parseCommonConfig(AbstractConfiguration conf)
    throws ConfigurationException {
  LoggingAuditClientConfig loggingAuditClientConfig = new LoggingAuditClientConfig();
  try {
    loggingAuditClientConfig.setStage(LoggingAuditStage
        .valueOf(conf.getString(LoggingAuditClientConfigDef.STAGE).toUpperCase()));
  } catch (Exception e) {
    throw new ConfigurationException("STAGE is not properly set!");
  }

  if (conf.containsKey(LoggingAuditClientConfigDef.DEFAULT_ENABLE_AUDIT_FOR_ALL_TOPICS)) {
    loggingAuditClientConfig.setEnableAuditForAllTopicsByDefault(
        conf.getBoolean(LoggingAuditClientConfigDef.DEFAULT_ENABLE_AUDIT_FOR_ALL_TOPICS));
  }

  if (conf.containsKey(LoggingAuditClientConfigDef.QUEUE_SIZE)) {
    loggingAuditClientConfig.setQueueSize(conf.getInt(LoggingAuditClientConfigDef.QUEUE_SIZE));
  }
  if (conf.containsKey(LoggingAuditClientConfigDef.ENQUEUE_WAIT_IN_MILLISECONDS)) {
    loggingAuditClientConfig.setEnqueueWaitInMilliseconds(
        conf.getInt(LoggingAuditClientConfigDef.ENQUEUE_WAIT_IN_MILLISECONDS));
  }
  return loggingAuditClientConfig;
}
 

public static AuditConfig parseAuditConfig(AbstractConfiguration conf)
    throws ConfigurationException {
  AuditConfig topicAuditConfig = new AuditConfig();
  try {
    if (conf.containsKey(LoggingAuditClientConfigDef.SAMPLING_RATE)) {
      topicAuditConfig
          .setSamplingRate(conf.getDouble(LoggingAuditClientConfigDef.SAMPLING_RATE));
    }
    if (conf.containsKey(LoggingAuditClientConfigDef.START_AT_CURRENT_STAGE)) {
      topicAuditConfig.setStartAtCurrentStage(
          conf.getBoolean(LoggingAuditClientConfigDef.START_AT_CURRENT_STAGE));
    }
    if (conf.containsKey(LoggingAuditClientConfigDef.STOP_AT_CURRENT_STAGE)) {
      topicAuditConfig
          .setStopAtCurrentStage(
              conf.getBoolean(LoggingAuditClientConfigDef.STOP_AT_CURRENT_STAGE));
    }
    return topicAuditConfig;
  } catch (Exception e) {
    throw new ConfigurationException("Can't create TopicAuditConfig from configuration.", e);
  }
}
 

private static KubeConfig parseKubeConfig(PropertiesConfiguration configHeader) throws ConfigurationException {
  KubeConfig config = new KubeConfig();
  AbstractConfiguration subsetConfig = new SubsetConfiguration(configHeader,
      SingerConfigDef.SINGER_KUBE_CONFIG_PREFIX);
  if (subsetConfig.containsKey(SingerConfigDef.KUBE_POLL_FREQUENCY_SECONDS)) {
    config.setPollFrequencyInSeconds(
        subsetConfig.getInt(SingerConfigDef.KUBE_POLL_FREQUENCY_SECONDS));
  }

  if (subsetConfig.containsKey(SingerConfigDef.KUBE_POD_LOG_DIR)) {
    String logDirectory = subsetConfig.getString(SingerConfigDef.KUBE_POD_LOG_DIR);
    // normalize path before setting the property
    logDirectory = new File(logDirectory).toPath().normalize().toString() + "/";
    if (!logDirectory.isEmpty() && !logDirectory.endsWith("/")) {
      logDirectory += "/";
    }
    config.setPodLogDirectory(logDirectory);
  }

  if (subsetConfig.containsKey(SingerConfigDef.KUBE_DEFAULT_DELETION_TIMEOUT)) {
    config.setDefaultDeletionTimeoutInSeconds(
        subsetConfig.getInt(SingerConfigDef.KUBE_DEFAULT_DELETION_TIMEOUT));
  }
  return config;
}
 

public static RealpinWriterConfig parseRealpinWriterConfig(AbstractConfiguration configuration) {
  configuration.setThrowExceptionOnMissing(true);
  String topic = configuration.getString(SingerConfigDef.TOPIC);
  String objectTypeString = configuration.getString(SingerConfigDef.REALPIN_OBJECT_TYPE);
  RealpinObjectType objectType = RealpinObjectType.valueOf(objectTypeString.toUpperCase());
  String serverSetPath = configuration.getString(SingerConfigDef.REALPIN_SERVERSET_PATH);

  RealpinWriterConfig writer = new RealpinWriterConfig(topic, objectType, serverSetPath);
  if (configuration.containsKey(SingerConfigDef.REALPIN_TIMEOUT_MS)) {
    writer.setTimeoutMs(configuration.getInt(SingerConfigDef.REALPIN_TIMEOUT_MS));
  }
  if (configuration.containsKey(SingerConfigDef.REALPIN_RETRIES)) {
    writer.setRetries(configuration.getInt(SingerConfigDef.REALPIN_RETRIES));
  }
  if (configuration.containsKey(SingerConfigDef.REALPIN_HOST_LIMIT)) {
    writer.setHostLimit(configuration.getInt(SingerConfigDef.REALPIN_HOST_LIMIT));
  }
  if (configuration.containsKey(SingerConfigDef.REALPIN_MAX_WAITERS)) {
    writer.setMaxWaiters(configuration.getInt(SingerConfigDef.REALPIN_MAX_WAITERS));
  }
  if (configuration.containsKey(SingerConfigDef.REALPIN_TTL)) {
    writer.setTtl(configuration.getInt(SingerConfigDef.REALPIN_TTL));
  }
  return writer;
}
 

private static void duplicateCseConfigToServicecomb(AbstractConfiguration source) {
  Iterator<String> keys = source.getKeys();
  while (keys.hasNext()) {
    String key = keys.next();
    if (!key.startsWith(CONFIG_CSE_PREFIX)) {
      continue;
    }

    String servicecombKey = CONFIG_SERVICECOMB_PREFIX + key.substring(key.indexOf(".") + 1);
    if (!source.containsKey(servicecombKey)) {
      source.addProperty(servicecombKey, source.getProperty(key));
    } else {
      LOGGER
          .warn(
              "Key {} with an ambiguous item {} exists, it's recommended to use only one of them.",
              key, servicecombKey);
    }
  }
}
 

public static Map<String, AuditConfig> parseAllAuditConfigs(AbstractConfiguration conf) {
  Map<String, AuditConfig> auditConfigs = new HashMap<>();
  if (conf != null && conf.containsKey(LoggingAuditClientConfigDef.AUDITED_TOPIC_NAMES)) {
    for (String name : conf.getStringArray(LoggingAuditClientConfigDef.AUDITED_TOPIC_NAMES)) {
      try {
        auditConfigs.put(name, parseAuditConfig(new SubsetConfiguration(conf, name + ".")));
      } catch (ConfigurationException e) {
        LOG.error("Can't parse TopicAuditConfig for {}", name);
      }
    }
  }
  return auditConfigs;
}
 

public static KafkaSenderConfig parseKafkaSenderConfig(AbstractConfiguration conf)
    throws ConfigurationException {
  KafkaSenderConfig kafkaSenderConfig = new KafkaSenderConfig();
  if (conf.containsKey(LoggingAuditClientConfigDef.KAFKA_TOPIC)) {
    kafkaSenderConfig.setTopic(conf.getString(LoggingAuditClientConfigDef.KAFKA_TOPIC));
  }
  if (conf.containsKey(LoggingAuditClientConfigDef.KAFKA_STOP_GRACE_PERIOD_IN_SECONDS)) {
    kafkaSenderConfig.setStopGracePeriodInSeconds(
        conf.getInt(LoggingAuditClientConfigDef.KAFKA_STOP_GRACE_PERIOD_IN_SECONDS));
  }
  kafkaSenderConfig.setKafkaProducerConfig(parseProducerConfig(
      new SubsetConfiguration(conf, LoggingAuditClientConfigDef.KAFKA_PRODUCER_CONFIG_PREFIX)));
  return kafkaSenderConfig;
}
 

/**
 * Singer can restart itself if # of failures exceeds threshold, and in daily
 * cadence. The following is singer restart related configuration:
 *
 * singer.restart.onFailures=true singer.restart.numberOfFailuresAllowed=100
 * singer.restart.daily=true singer.restart.dailyRestartUtcTimeRangeBegin=02:30
 * singer.restart.dailyRestartUtcTimeRangeEnd=03:30
 */
private static SingerRestartConfig parseSingerRestartConfig(PropertiesConfiguration configHeader) throws ConfigurationException {
  SingerRestartConfig restartConfig = new SingerRestartConfig();
  AbstractConfiguration subsetConfig = new SubsetConfiguration(configHeader,
      SingerConfigDef.SINGER_RESTART_PREFIX);

  if (subsetConfig.containsKey(SingerConfigDef.ON_FAILURES)) {
    restartConfig.restartOnFailures = subsetConfig.getBoolean(SingerConfigDef.ON_FAILURES);
  }
  if (subsetConfig.containsKey(SingerConfigDef.NUMBER_OF_FAILURES_ALLOWED)) {
    restartConfig.numOfFailuesAllowed = subsetConfig
        .getInt(SingerConfigDef.NUMBER_OF_FAILURES_ALLOWED);
  }
  if (subsetConfig.containsKey(SingerConfigDef.DAILY_RESTART_FLAG)) {
    restartConfig.restartDaily = subsetConfig.getBoolean(SingerConfigDef.DAILY_RESTART_FLAG);
  }

  if (restartConfig.restartDaily) {
    if (!subsetConfig.containsKey(SingerConfigDef.DAILY_RESTART_TIME_BEGIN)
        || !subsetConfig.containsKey(SingerConfigDef.DAILY_RESTART_TIME_END)) {
      throw new ConfigurationException("Daily restart time range is not set correctly");
    }

    restartConfig.dailyRestartUtcTimeRangeBegin = subsetConfig
        .getString(SingerConfigDef.DAILY_RESTART_TIME_BEGIN);
    restartConfig.dailyRestartUtcTimeRangeEnd = subsetConfig
        .getString(SingerConfigDef.DAILY_RESTART_TIME_END);
    Date startTime = SingerUtils.convertToDate(restartConfig.dailyRestartUtcTimeRangeBegin);
    Date endTime = SingerUtils.convertToDate(restartConfig.dailyRestartUtcTimeRangeEnd);
    if (endTime.compareTo(startTime) <= 0) {
      throw new ConfigurationException("Daily restart end time is not later than start time");
    }
  }
  return restartConfig;
}
 

public static SingerLogConfig parseLogConfig(String logName,
                                             AbstractConfiguration logConfiguration) throws ConfigurationException {
  logConfiguration.setThrowExceptionOnMissing(true);

  String local_dir = null;
  if (logConfiguration.containsKey("logDir")) {
    local_dir = logConfiguration.getString("logDir");
  } else if (logConfiguration.containsKey("local_dir")) {
    local_dir = logConfiguration.getString("local_dir");
  } else {
    throw new ConfigurationException("missing logDir/local_dir");
  }
  if (local_dir != null && local_dir.endsWith("/")) {
    local_dir = local_dir.substring(0, local_dir.length() - 1);
  }

  String logfile_regex;
  if (logConfiguration.containsKey("logStreamRegex")) {
    logfile_regex = logConfiguration.getString("logStreamRegex");
  } else if (logConfiguration.containsKey("logfile_regex")) {
    logfile_regex = logConfiguration.getString("logfile_regex");
  } else {
    throw new ConfigurationException("missing logStreamRegex/logfile_regex");
  }

  LogStreamProcessorConfig processorConfig = parseLogStreamProcessorConfig(
      new SubsetConfiguration(logConfiguration, "processor."));
  LogStreamReaderConfig readerConfig = parseLogStreamReaderConfig(
      new SubsetConfiguration(logConfiguration, "reader."));
  LogStreamWriterConfig writerConfig = parseLogStreamWriterConfig(
      new SubsetConfiguration(logConfiguration, "writer."));

  // initialize the optional fields
  logConfiguration.setThrowExceptionOnMissing(false);
  String logDecider = logConfiguration.getString("logDecider");
  SingerLogConfig config = new SingerLogConfig(logName, local_dir, logfile_regex, processorConfig,
      readerConfig, writerConfig);
  config.setLogDecider(logDecider);
  if (logConfiguration.containsKey("enableHeadersInjector")){
    boolean enableHeadersInjector = logConfiguration.getBoolean("enableHeadersInjector");
    config.setEnableHeadersInjector(enableHeadersInjector);
    if (enableHeadersInjector && logConfiguration.containsKey("headersInjectorClass")){
      config.setHeadersInjectorClass(logConfiguration.getString("headersInjectorClass"));
    }
  }

  FileNameMatchMode matchMode = FileNameMatchMode.PREFIX;
  String matchModeStr = logConfiguration.getString("logFileMatchMode");
  if (matchModeStr != null) {
    matchModeStr = matchModeStr.toLowerCase();
    if (matchModeStr.equals("exact")) {
      matchMode = FileNameMatchMode.EXACT;
    } else if (!matchModeStr.equals("prefix")) {
      throw new ConfigurationException("Invalid logFileNameMatchMode : " + matchModeStr);
    }
  }
  config.setFilenameMatchMode(matchMode);

  if (logConfiguration.containsKey(SingerConfigDef.LOG_RETENTION_SECONDS)) {
    config
        .setLogRetentionInSeconds(logConfiguration.getInt(SingerConfigDef.LOG_RETENTION_SECONDS));
  }

  if (logConfiguration.containsKey("enableLoggingAudit")){
    boolean enableLoggingAudit = logConfiguration.getBoolean("enableLoggingAudit");
    config.setEnableLoggingAudit(enableLoggingAudit);
    try {
      AuditConfig auditConfig = ConfigUtils.parseAuditConfig(new SubsetConfiguration(
          logConfiguration, "loggingaudit."));
      config.setAuditConfig(auditConfig);
    } catch(ConfigurationException e){
       LOG.error("TopicAuditConfig is not configured correctly for {}", logName, e);
       config.setEnableLoggingAudit(false);
    }
  }

  return config;
}
 

private static KafkaWriterConfig parseKafkaWriterConfig(AbstractConfiguration kafkaWriterConfiguration) throws ConfigurationException {
  kafkaWriterConfiguration.setThrowExceptionOnMissing(true);
  String topic;
  try {
    topic = kafkaWriterConfiguration.getString(SingerConfigDef.TOPIC);
  } catch (Exception x) {
    throw new ConfigurationException("KafkaWriter topic is required for Singer Configuration");
  }
  KafkaProducerConfig producerConfig = parseProducerConfig(
      new SubsetConfiguration(kafkaWriterConfiguration, SingerConfigDef.PRODUCER_CONFIG_PREFIX));

  String auditTopic = null;

  boolean auditingEnabled = false;
  if (kafkaWriterConfiguration.containsKey(SingerConfigDef.AUDITING_ENABLED)) {
    auditingEnabled = kafkaWriterConfiguration.getBoolean(SingerConfigDef.AUDITING_ENABLED);

    if (auditingEnabled) {
      if (!kafkaWriterConfiguration.containsKey(SingerConfigDef.AUDIT_TOPIC)) {
        throw new ConfigurationException("Auditing enabled but missing audit topic");
      } else {
        auditTopic = kafkaWriterConfiguration.getString(SingerConfigDef.AUDIT_TOPIC);
      }
    }
  }

  boolean skipNoLeaderPartitions = false;
  if (kafkaWriterConfiguration.containsKey(SingerConfigDef.SKIP_NO_LEADER_PARTITIONS)) {
    skipNoLeaderPartitions = kafkaWriterConfiguration
        .getBoolean(SingerConfigDef.SKIP_NO_LEADER_PARTITIONS);
  }

  int writeTimeoutInSeconds = 60;
  if (kafkaWriterConfiguration.containsKey(SingerConfigDef.KAFKA_WRITE_TIMEOUT_IN_SECONDS)) {
    writeTimeoutInSeconds = kafkaWriterConfiguration
        .getInt(SingerConfigDef.KAFKA_WRITE_TIMEOUT_IN_SECONDS);
  }

  KafkaWriterConfig writerConfig = new KafkaWriterConfig(topic, producerConfig);
  writerConfig.setAuditTopic(auditTopic);
  writerConfig.setAuditingEnabled(auditingEnabled);
  writerConfig.setSkipNoLeaderPartitions(skipNoLeaderPartitions);
  writerConfig.setWriteTimeoutInSeconds(writeTimeoutInSeconds);
  return writerConfig;
}