Java Code Examples for org.apache.commons.configuration.AbstractConfiguration#getString()

private static KubeConfig parseKubeConfig(PropertiesConfiguration configHeader) throws ConfigurationException {
  KubeConfig config = new KubeConfig();
  AbstractConfiguration subsetConfig = new SubsetConfiguration(configHeader,
      SingerConfigDef.SINGER_KUBE_CONFIG_PREFIX);
  if (subsetConfig.containsKey(SingerConfigDef.KUBE_POLL_FREQUENCY_SECONDS)) {
    config.setPollFrequencyInSeconds(
        subsetConfig.getInt(SingerConfigDef.KUBE_POLL_FREQUENCY_SECONDS));
  }

  if (subsetConfig.containsKey(SingerConfigDef.KUBE_POD_LOG_DIR)) {
    String logDirectory = subsetConfig.getString(SingerConfigDef.KUBE_POD_LOG_DIR);
    // normalize path before setting the property
    logDirectory = new File(logDirectory).toPath().normalize().toString() + "/";
    if (!logDirectory.isEmpty() && !logDirectory.endsWith("/")) {
      logDirectory += "/";
    }
    config.setPodLogDirectory(logDirectory);
  }

  if (subsetConfig.containsKey(SingerConfigDef.KUBE_DEFAULT_DELETION_TIMEOUT)) {
    config.setDefaultDeletionTimeoutInSeconds(
        subsetConfig.getInt(SingerConfigDef.KUBE_DEFAULT_DELETION_TIMEOUT));
  }
  return config;
}
 

public static RealpinWriterConfig parseRealpinWriterConfig(AbstractConfiguration configuration) {
  configuration.setThrowExceptionOnMissing(true);
  String topic = configuration.getString(SingerConfigDef.TOPIC);
  String objectTypeString = configuration.getString(SingerConfigDef.REALPIN_OBJECT_TYPE);
  RealpinObjectType objectType = RealpinObjectType.valueOf(objectTypeString.toUpperCase());
  String serverSetPath = configuration.getString(SingerConfigDef.REALPIN_SERVERSET_PATH);

  RealpinWriterConfig writer = new RealpinWriterConfig(topic, objectType, serverSetPath);
  if (configuration.containsKey(SingerConfigDef.REALPIN_TIMEOUT_MS)) {
    writer.setTimeoutMs(configuration.getInt(SingerConfigDef.REALPIN_TIMEOUT_MS));
  }
  if (configuration.containsKey(SingerConfigDef.REALPIN_RETRIES)) {
    writer.setRetries(configuration.getInt(SingerConfigDef.REALPIN_RETRIES));
  }
  if (configuration.containsKey(SingerConfigDef.REALPIN_HOST_LIMIT)) {
    writer.setHostLimit(configuration.getInt(SingerConfigDef.REALPIN_HOST_LIMIT));
  }
  if (configuration.containsKey(SingerConfigDef.REALPIN_MAX_WAITERS)) {
    writer.setMaxWaiters(configuration.getInt(SingerConfigDef.REALPIN_MAX_WAITERS));
  }
  if (configuration.containsKey(SingerConfigDef.REALPIN_TTL)) {
    writer.setTtl(configuration.getInt(SingerConfigDef.REALPIN_TTL));
  }
  return writer;
}
 

private static LogStreamReaderConfig parseLogStreamReaderConfig(AbstractConfiguration readerConfiguration) throws ConfigurationException {
  readerConfiguration.setThrowExceptionOnMissing(true);
  String readerTypeString = readerConfiguration.getString("type");
  ReaderType type = ReaderType.valueOf(readerTypeString.toUpperCase());

  LogStreamReaderConfig readerConfig = new LogStreamReaderConfig(type);
  if (type.equals(ReaderType.THRIFT)) {
    ThriftReaderConfig thriftReaderConfig = parseThriftReaderConfig(
        new SubsetConfiguration(readerConfiguration, readerTypeString + "."));
    readerConfig.setThriftReaderConfig(thriftReaderConfig);
  } else if (type.equals(ReaderType.TEXT)) {
    TextReaderConfig textReaderConfig = parseTextReaderConfig(
        new SubsetConfiguration(readerConfiguration, readerTypeString + "."));
    readerConfig.setTextReaderConfig(textReaderConfig);
  }

  return readerConfig;
}
 

private void initZuul() throws Exception {
    LOGGER.info("Starting Groovy Filter file manager");
    final AbstractConfiguration config = ConfigurationManager.getConfigInstance();
    final String preFiltersPath = config.getString(Constants.ZUUL_FILTER_PRE_PATH);
    final String postFiltersPath = config.getString(Constants.ZUUL_FILTER_POST_PATH);
    final String routeFiltersPath = config.getString(Constants.ZUUL_FILTER_ROUTE_PATH);
    final String errorFiltersPath = config.getString(Constants.ZUUL_FILTER_ERROR_PATH);
    final String customPath = config.getString(Constants.Zuul_FILTER_CUSTOM_PATH);

    //load local filter files
    FilterLoader.getInstance().setCompiler(new GroovyCompiler());
    FilterFileManager.setFilenameFilter(new GroovyFileFilter());
    if (customPath == null) {
        FilterFileManager.init(5, preFiltersPath, postFiltersPath, routeFiltersPath, errorFiltersPath);
    } else {
        FilterFileManager.init(5, preFiltersPath, postFiltersPath, routeFiltersPath, errorFiltersPath, customPath);
    }
    //load filters in DB
    startZuulFilterPoller();
    LOGGER.info("Groovy Filter file manager started");
}
 

/**
 * Gets the template builder based on the configuration settings
 * @return the template builder
 */
static TemplateBuilder getTemplateBuilder() {
    AbstractConfiguration config = ConfigurationWrapper.getConfig();

    String type = config.getString("maestro.reports.db.type", "internal");

    if (type.equals("internal")) {
        return new InternalDatabaseBuilder();
    }

    if (type.equals("external")) {
        return new ExternalDatabaseBuilder();
    }

    return new EnvironmentDatabaseBuilder();
}
 

/**
 * Constructor
 * @param maestroURL maestro_broker URL
 * @param peerInfo maestro peer information
 * @throws MaestroException if unable to create agent instance
 */
public MaestroAgent(final String maestroURL, final PeerInfo peerInfo) throws MaestroException {
    super(maestroURL, peerInfo);

    final AbstractConfiguration config = ConfigurationWrapper.getConfig();
    String pathStr = config.getString("agent.ext.path.override", null);

    if (pathStr == null){
        pathStr = Constants.HOME_DIR + "ext" + File.separator + "requests";
    }

    File defaultExtPointFile = new File(pathStr);
    if (defaultExtPointFile.exists()) {
        extensionPoints.add(new ExtensionPoint(defaultExtPointFile, false));
    }
    else  {
        logger.warn("The extension point at {} does not exist", defaultExtPointFile.getPath());
    }

    final String defaultSourceDir = FileUtils.getTempDirectoryPath() + File.separator + "maestro-agent-work";

    sourceRoot = config.getString("maestro.agent.source.root", defaultSourceDir);
    groovyHandler = new GroovyHandler(super.getClient());
}
 

@Override
public void init(AbstractConfiguration config) {
    if (!config.getString("redis.type").equals("single")) {
        throw new IllegalStateException("RedisSyncSingleStorageImpl class can only be used with single redis setup, but redis.type value is " + config.getString("redis.type"));
    }

    List<String> address = parseRedisAddress(config.getString("redis.address"), 6379);
    int databaseNumber = config.getInt("redis.database", 0);
    String password = StringUtils.isNotEmpty(config.getString("redis.password")) ? config.getString("redis.password") + "@" : "";

    // lettuce
    RedisURI lettuceURI = RedisURI.create("redis://" + password + address.get(0) + "/" + databaseNumber);
    this.lettuce = RedisClient.create(lettuceURI);
    this.lettuceConn = this.lettuce.connect();

    // params
    initParams(config);
}
 

@Override
public void init(AbstractConfiguration config) {
    if (!config.getString("redis.type").equals("sentinel")) {
        throw new IllegalStateException("RedisSyncSingleStorageImpl class can only be used with sentinel redis setup, but redis.type value is " + config.getString("redis.type"));
    }

    List<String> address = parseRedisAddress(config.getString("redis.address"), 26379);
    int databaseNumber = config.getInt("redis.database", 0);
    String password = StringUtils.isNotEmpty(config.getString("redis.password")) ? config.getString("redis.password") + "@" : "";
    String masterId = config.getString("redis.master");

    // lettuce
    RedisURI lettuceURI = RedisURI.create("redis-sentinel://" + password + String.join(",", address) + "/" + databaseNumber + "#" + masterId);
    this.lettuceSentinel = RedisClient.create(lettuceURI);
    this.lettuceSentinelConn = MasterSlave.connect(this.lettuceSentinel, new Utf8StringCodec(), lettuceURI);
    this.lettuceSentinelConn.setReadFrom(ReadFrom.valueOf(config.getString("redis.read")));

    // params
    initParams(config);
}
 

@Override
public void init(AbstractConfiguration config) {
    if (!config.getString("redis.type").equals("master_slave")) {
        throw new IllegalStateException("RedisSyncSingleStorageImpl class can only be used with master slave redis setup, but redis.type value is " + config.getString("redis.type"));
    }

    List<String> address = parseRedisAddress(config.getString("redis.address"), 6379);
    int databaseNumber = config.getInt("redis.database", 0);
    String password = StringUtils.isNotEmpty(config.getString("redis.password")) ? config.getString("redis.password") + "@" : "";

    // lettuce
    RedisURI lettuceURI = RedisURI.create("redis://" + password + address.get(0) + "/" + databaseNumber);
    this.lettuceMasterSlave = RedisClient.create(lettuceURI);
    this.lettuceMasterSlaveConn = MasterSlave.connect(this.lettuceMasterSlave, new Utf8StringCodec(), lettuceURI);
    this.lettuceMasterSlaveConn.setReadFrom(ReadFrom.valueOf(config.getString("redis.read")));

    // params
    initParams(config);
}
 

/**
 * Singer can restart itself if # of failures exceeds threshold, and in daily
 * cadence. The following is singer restart related configuration:
 *
 * singer.restart.onFailures=true singer.restart.numberOfFailuresAllowed=100
 * singer.restart.daily=true singer.restart.dailyRestartUtcTimeRangeBegin=02:30
 * singer.restart.dailyRestartUtcTimeRangeEnd=03:30
 */
private static SingerRestartConfig parseSingerRestartConfig(PropertiesConfiguration configHeader) throws ConfigurationException {
  SingerRestartConfig restartConfig = new SingerRestartConfig();
  AbstractConfiguration subsetConfig = new SubsetConfiguration(configHeader,
      SingerConfigDef.SINGER_RESTART_PREFIX);

  if (subsetConfig.containsKey(SingerConfigDef.ON_FAILURES)) {
    restartConfig.restartOnFailures = subsetConfig.getBoolean(SingerConfigDef.ON_FAILURES);
  }
  if (subsetConfig.containsKey(SingerConfigDef.NUMBER_OF_FAILURES_ALLOWED)) {
    restartConfig.numOfFailuesAllowed = subsetConfig
        .getInt(SingerConfigDef.NUMBER_OF_FAILURES_ALLOWED);
  }
  if (subsetConfig.containsKey(SingerConfigDef.DAILY_RESTART_FLAG)) {
    restartConfig.restartDaily = subsetConfig.getBoolean(SingerConfigDef.DAILY_RESTART_FLAG);
  }

  if (restartConfig.restartDaily) {
    if (!subsetConfig.containsKey(SingerConfigDef.DAILY_RESTART_TIME_BEGIN)
        || !subsetConfig.containsKey(SingerConfigDef.DAILY_RESTART_TIME_END)) {
      throw new ConfigurationException("Daily restart time range is not set correctly");
    }

    restartConfig.dailyRestartUtcTimeRangeBegin = subsetConfig
        .getString(SingerConfigDef.DAILY_RESTART_TIME_BEGIN);
    restartConfig.dailyRestartUtcTimeRangeEnd = subsetConfig
        .getString(SingerConfigDef.DAILY_RESTART_TIME_END);
    Date startTime = SingerUtils.convertToDate(restartConfig.dailyRestartUtcTimeRangeBegin);
    Date endTime = SingerUtils.convertToDate(restartConfig.dailyRestartUtcTimeRangeEnd);
    if (endTime.compareTo(startTime) <= 0) {
      throw new ConfigurationException("Daily restart end time is not later than start time");
    }
  }
  return restartConfig;
}
 

private static LogStreamWriterConfig parseLogStreamWriterConfig(AbstractConfiguration writerConfiguration) throws ConfigurationException {
  writerConfiguration.setThrowExceptionOnMissing(true);
  String writerTypeString = writerConfiguration.getString("type");
  WriterType type = WriterType.valueOf(writerTypeString.toUpperCase());
  LogStreamWriterConfig writerConfig = new LogStreamWriterConfig(type);
  switch (type) {
  case KAFKA08:
  case KAFKA:
    writerConfig.setKafkaWriterConfig(parseKafkaWriterConfig(
        new SubsetConfiguration(writerConfiguration, writerTypeString + ".")));
    return writerConfig;
  case REALPIN:
    writerConfig.setRealpinWriterConfig(parseRealpinWriterConfig(
        new SubsetConfiguration(writerConfiguration, writerTypeString + ".")));
    return writerConfig;
  case DUMMY:
    writerConfig.setDummyWriteConfig(parseDummyWriterConfig(
        new SubsetConfiguration(writerConfiguration, writerTypeString + ".")));
    return writerConfig;
  case PULSAR:
    writerConfig.setPulsarWriterConfig(parsePulsarWriterConfig(
        new SubsetConfiguration(writerConfiguration, writerTypeString + ".")));
    return writerConfig;
  default:
    throw new ConfigurationException("Unsupported log writer type.");
  }
}
 

@Override
public void init(AbstractConfiguration config) {
    if (!config.getString("redis.type").equals("cluster")) {
        throw new IllegalStateException("RedisSyncSingleStorageImpl class can only be used with cluster redis setup, but redis.type value is " + config.getString("redis.type"));
    }

    List<String> address = parseRedisAddress(config.getString("redis.address"), 6379);
    int databaseNumber = config.getInt("redis.database", 0);
    String password = StringUtils.isNotEmpty(config.getString("redis.password")) ? config.getString("redis.password") + "@" : "";

    // lettuce
    RedisURI lettuceURI = RedisURI.create("redis://" + password + address.get(0) + "/" + databaseNumber);
    this.lettuceCluster = RedisClusterClient.create(lettuceURI);
    this.lettuceCluster.setOptions(ClusterClientOptions.builder()
            .topologyRefreshOptions(ClusterTopologyRefreshOptions.builder()
                    .enablePeriodicRefresh(config.getBoolean("redis.cluster.periodicRefreshEnabled", ClusterTopologyRefreshOptions.DEFAULT_PERIODIC_REFRESH_ENABLED))
                    .refreshPeriod(config.getLong("redis.cluster.refreshPeriod", ClusterTopologyRefreshOptions.DEFAULT_REFRESH_PERIOD), TimeUnit.SECONDS)
                    .closeStaleConnections(config.getBoolean("redis.cluster.closeStaleConnections", ClusterTopologyRefreshOptions.DEFAULT_CLOSE_STALE_CONNECTIONS))
                    .build())
            .validateClusterNodeMembership(config.getBoolean("redis.cluster.validateClusterNodeMembership", ClusterClientOptions.DEFAULT_VALIDATE_CLUSTER_MEMBERSHIP))
            .maxRedirects(config.getInt("redis.cluster.refreshPeriod", ClusterClientOptions.DEFAULT_MAX_REDIRECTS))
            .build());
    this.lettuceClusterConn = this.lettuceCluster.connect();
    this.lettuceClusterConn.setReadFrom(ReadFrom.valueOf(config.getString("redis.read")));

    // params
    initParams(config);
}
 

public static SingerLogConfig parseLogConfig(String logName,
                                             AbstractConfiguration logConfiguration) throws ConfigurationException {
  logConfiguration.setThrowExceptionOnMissing(true);

  String local_dir = null;
  if (logConfiguration.containsKey("logDir")) {
    local_dir = logConfiguration.getString("logDir");
  } else if (logConfiguration.containsKey("local_dir")) {
    local_dir = logConfiguration.getString("local_dir");
  } else {
    throw new ConfigurationException("missing logDir/local_dir");
  }
  if (local_dir != null && local_dir.endsWith("/")) {
    local_dir = local_dir.substring(0, local_dir.length() - 1);
  }

  String logfile_regex;
  if (logConfiguration.containsKey("logStreamRegex")) {
    logfile_regex = logConfiguration.getString("logStreamRegex");
  } else if (logConfiguration.containsKey("logfile_regex")) {
    logfile_regex = logConfiguration.getString("logfile_regex");
  } else {
    throw new ConfigurationException("missing logStreamRegex/logfile_regex");
  }

  LogStreamProcessorConfig processorConfig = parseLogStreamProcessorConfig(
      new SubsetConfiguration(logConfiguration, "processor."));
  LogStreamReaderConfig readerConfig = parseLogStreamReaderConfig(
      new SubsetConfiguration(logConfiguration, "reader."));
  LogStreamWriterConfig writerConfig = parseLogStreamWriterConfig(
      new SubsetConfiguration(logConfiguration, "writer."));

  // initialize the optional fields
  logConfiguration.setThrowExceptionOnMissing(false);
  String logDecider = logConfiguration.getString("logDecider");
  SingerLogConfig config = new SingerLogConfig(logName, local_dir, logfile_regex, processorConfig,
      readerConfig, writerConfig);
  config.setLogDecider(logDecider);
  if (logConfiguration.containsKey("enableHeadersInjector")){
    boolean enableHeadersInjector = logConfiguration.getBoolean("enableHeadersInjector");
    config.setEnableHeadersInjector(enableHeadersInjector);
    if (enableHeadersInjector && logConfiguration.containsKey("headersInjectorClass")){
      config.setHeadersInjectorClass(logConfiguration.getString("headersInjectorClass"));
    }
  }

  FileNameMatchMode matchMode = FileNameMatchMode.PREFIX;
  String matchModeStr = logConfiguration.getString("logFileMatchMode");
  if (matchModeStr != null) {
    matchModeStr = matchModeStr.toLowerCase();
    if (matchModeStr.equals("exact")) {
      matchMode = FileNameMatchMode.EXACT;
    } else if (!matchModeStr.equals("prefix")) {
      throw new ConfigurationException("Invalid logFileNameMatchMode : " + matchModeStr);
    }
  }
  config.setFilenameMatchMode(matchMode);

  if (logConfiguration.containsKey(SingerConfigDef.LOG_RETENTION_SECONDS)) {
    config
        .setLogRetentionInSeconds(logConfiguration.getInt(SingerConfigDef.LOG_RETENTION_SECONDS));
  }

  if (logConfiguration.containsKey("enableLoggingAudit")){
    boolean enableLoggingAudit = logConfiguration.getBoolean("enableLoggingAudit");
    config.setEnableLoggingAudit(enableLoggingAudit);
    try {
      AuditConfig auditConfig = ConfigUtils.parseAuditConfig(new SubsetConfiguration(
          logConfiguration, "loggingaudit."));
      config.setAuditConfig(auditConfig);
    } catch(ConfigurationException e){
       LOG.error("TopicAuditConfig is not configured correctly for {}", logName, e);
       config.setEnableLoggingAudit(false);
    }
  }

  return config;
}
 

private static KafkaWriterConfig parseKafkaWriterConfig(AbstractConfiguration kafkaWriterConfiguration) throws ConfigurationException {
  kafkaWriterConfiguration.setThrowExceptionOnMissing(true);
  String topic;
  try {
    topic = kafkaWriterConfiguration.getString(SingerConfigDef.TOPIC);
  } catch (Exception x) {
    throw new ConfigurationException("KafkaWriter topic is required for Singer Configuration");
  }
  KafkaProducerConfig producerConfig = parseProducerConfig(
      new SubsetConfiguration(kafkaWriterConfiguration, SingerConfigDef.PRODUCER_CONFIG_PREFIX));

  String auditTopic = null;

  boolean auditingEnabled = false;
  if (kafkaWriterConfiguration.containsKey(SingerConfigDef.AUDITING_ENABLED)) {
    auditingEnabled = kafkaWriterConfiguration.getBoolean(SingerConfigDef.AUDITING_ENABLED);

    if (auditingEnabled) {
      if (!kafkaWriterConfiguration.containsKey(SingerConfigDef.AUDIT_TOPIC)) {
        throw new ConfigurationException("Auditing enabled but missing audit topic");
      } else {
        auditTopic = kafkaWriterConfiguration.getString(SingerConfigDef.AUDIT_TOPIC);
      }
    }
  }

  boolean skipNoLeaderPartitions = false;
  if (kafkaWriterConfiguration.containsKey(SingerConfigDef.SKIP_NO_LEADER_PARTITIONS)) {
    skipNoLeaderPartitions = kafkaWriterConfiguration
        .getBoolean(SingerConfigDef.SKIP_NO_LEADER_PARTITIONS);
  }

  int writeTimeoutInSeconds = 60;
  if (kafkaWriterConfiguration.containsKey(SingerConfigDef.KAFKA_WRITE_TIMEOUT_IN_SECONDS)) {
    writeTimeoutInSeconds = kafkaWriterConfiguration
        .getInt(SingerConfigDef.KAFKA_WRITE_TIMEOUT_IN_SECONDS);
  }

  KafkaWriterConfig writerConfig = new KafkaWriterConfig(topic, producerConfig);
  writerConfig.setAuditTopic(auditTopic);
  writerConfig.setAuditingEnabled(auditingEnabled);
  writerConfig.setSkipNoLeaderPartitions(skipNoLeaderPartitions);
  writerConfig.setWriteTimeoutInSeconds(writeTimeoutInSeconds);
  return writerConfig;
}
 

private static DummyWriteConfig parseDummyWriterConfig(AbstractConfiguration configuration) {
  configuration.setThrowExceptionOnMissing(true);
  String topic = configuration.getString(SingerConfigDef.TOPIC);
  return new DummyWriteConfig(topic);
}
 

@Override
public void init(AbstractConfiguration config) {
    this.allowDollar = config.getBoolean("allowDollar", true);
    this.deniedTopic = config.getString("deniedTopic", null);
}