Java Code Examples for org.apache.commons.configuration.PropertiesConfiguration#getInt()

public static void main(String[] args) throws Exception {
  final PropertiesConfiguration config = TerrapinUtil.readPropertiesExitOnFailure(
      System.getProperties().getProperty("terrapin.config", "thrift.properties"));

  OstrichStatsReceiver statsReceiver = new OstrichStatsReceiver(Stats.get(""));
  int listenPort = config.getInt("thrift_port", 9090);
  TerrapinServiceImpl serviceImpl = new TerrapinServiceImpl(config,
      (List) config.getList("cluster_list"));
  Service<byte[], byte[]> service = new TerrapinService.Service(serviceImpl,
      new TBinaryProtocol.Factory());
  Server server = ServerBuilder.safeBuild(
      service,
      ServerBuilder.get()
          .name("TERRAPIN_THRIFT")
          .codec(ThriftServerFramedCodec.get())
          .hostConnectionMaxIdleTime(Duration.apply(1, TimeUnit.MINUTES))
          .maxConcurrentRequests(3000)
          .reportTo(statsReceiver)
          .bindTo(new InetSocketAddress(listenPort)));
  new OstrichAdminService(config.getInt(Constants.OSTRICH_METRICS_PORT, 9999)).start();
  LOG.info("\n#######################################"
          + "\n#      Ready To Serve Requests.       #"
          + "\n#######################################");
}
 

/**
 * @param configuration Config settings containing settings such
 * @throws Exception
 */
public TerrapinServiceImpl(PropertiesConfiguration configuration,
                           List<String> clusterList) throws Exception {
  String zkQuorum = TerrapinUtil.getZKQuorumFromConf(configuration);
  Preconditions.checkNotNull(zkQuorum, "Zookeeper quorum should not be empty/null.");
  ZooKeeperClient zkClient = TerrapinUtil.getZooKeeperClient(zkQuorum, 30);
  this.clusterClientMap = Maps.newHashMapWithExpectedSize(clusterList.size());
  for (String clusterName : clusterList) {
    try {
      FileSetViewManager fsViewManager = new FileSetViewManager(zkClient, clusterName);
      LOG.info("Connecting to cluster " + clusterName + " on " + zkQuorum);
      TerrapinClient terrapinClient = new TerrapinClient(
          fsViewManager,
          clusterName,
          configuration.getInt(Constants.TERRAPIN_SERVER_TARGET_PORT,
              Constants.DEFAULT_THRIFT_PORT),
          configuration.getInt(Constants.CLIENT_CONNECT_TIMEOUT_MILLIS, 300),
          configuration.getInt(Constants.CLIENT_RPC_TIMEOUT_MILLIS, 500));
      clusterClientMap.put(clusterName, terrapinClient);
      LOG.info("Done.");
    } catch (Exception e) {
      LOG.warn("Could not connect to cluster " + clusterName, e);
      throw e;
    }
  }
}
 

public RedisPools(PropertiesConfiguration configuration, String host, int port,
                  boolean dequeueOnly) {
  int numGeneralConnections = configuration.getInt("BACKEND_CONNECTIONS_PER_SHARD");
  int generalSocketTimeoutMillis = (int) TimeUnit.SECONDS.toMillis(
      configuration.getInt("BACKEND_SOCKET_TIMEOUT_SECONDS"));
  int numMonitorConnections = configuration.getInt("MONITOR_CONNECTIONS_PER_SHARD", 3);
  int monitorSocketTimeoutMillis = (int) TimeUnit.SECONDS.toMillis(
      configuration.getInt("MONITOR_SOCKET_TIMEOUT_SECONDS", 10));
  int maxWaitMillis = configuration.getInt("BACKEND_CONNECTION_MAX_WAIT_MILLIS");
  this.host = host;
  this.port = port;
  this.dequeueOnly = dequeueOnly;
  this.generalRedisPool = createRedisPool(
      host, port, numGeneralConnections,
      maxWaitMillis, generalSocketTimeoutMillis);
  this.monitorRedisPool = createRedisPool(
      host, port, numMonitorConnections,
      maxWaitMillis, monitorSocketTimeoutMillis);
}
 

public MySQLDataSources(PropertiesConfiguration configuration, String host, int port,
                        String user, String passwd, boolean dequeueOnly) {
  int numGeneralConnections = configuration.getInt("BACKEND_CONNECTIONS_PER_SHARD");
  int generalSocketTimeoutMillis = (int) TimeUnit.SECONDS.toMillis(
      configuration.getInt("BACKEND_SOCKET_TIMEOUT_SECONDS"));
  int numMonitorConnections = configuration.getInt("MONITOR_CONNECTIONS_PER_SHARD", 3);
  int monitorSocketTimeoutMillis = (int) TimeUnit.SECONDS.toMillis(
      configuration.getInt("MONITOR_SOCKET_TIMEOUT_SECONDS", 10));
  int maxWaitMillis = configuration.getInt("BACKEND_CONNECTION_MAX_WAIT_MILLIS");
  this.generalDataSource = createDataSource(
      host, port, user, passwd, numGeneralConnections,
      maxWaitMillis, generalSocketTimeoutMillis);
  this.monitorDataSource = createDataSource(
      host, port, user, passwd, numMonitorConnections,
      maxWaitMillis, monitorSocketTimeoutMillis);
  this.host = host;
  this.port = port;
  this.user = user;
  this.passwd = passwd;
  this.dequeueOnly = new AtomicBoolean(dequeueOnly);
}
 

/**
 * Creates an instance of the PinLaterMySQLBackend.
 *
 * @param configuration          configuration parameters for the backend.
 * @param serverHostName         hostname of the PinLater server.
 * @param serverStartTimeMillis  start time of the PinLater server.
 */
public PinLaterMySQLBackend(PropertiesConfiguration configuration,
                            String serverHostName,
                            long serverStartTimeMillis) throws Exception {
  super(configuration, "MySQL", serverHostName, serverStartTimeMillis);
  this.configuration = Preconditions.checkNotNull(configuration);
  this.countLimit = configuration.getInt("MYSQL_COUNT_LIMIT");
  this.numDbPerQueue = configuration.getInt("MYSQL_NUM_DB_PER_QUEUE", 1);
  this.mySQLHealthMonitor = new MySQLHealthMonitor(new HashSet<String>());

  // Start the JobQueueMonitor scheduled task.
  this.queueMonitorService = Executors.newSingleThreadScheduledExecutor(
      new ThreadFactoryBuilder().setDaemon(true).setNameFormat("MySQLJobQueueMonitor-%d")
          .build());

  // Call Base class's initialization function to initialize the shardMap, futurePool and dequeue
  // semaphoreMap.
  initialize();
}
 

/**
 * For segments on disk.
 * <p>Index directory passed in should be top level segment directory.
 * <p>If segment metadata file exists in multiple segment version, load the one in highest segment version.
 */
public SegmentMetadataImpl(File indexDir)
    throws IOException {
  _indexDir = indexDir;
  PropertiesConfiguration segmentMetadataPropertiesConfiguration = getPropertiesConfiguration(indexDir);
  _columnMetadataMap = new HashMap<>();
  _allColumns = new HashSet<>();
  _schema = new Schema();

  init(segmentMetadataPropertiesConfiguration);
  File creationMetaFile = SegmentDirectoryPaths.findCreationMetaFile(indexDir);
  if (creationMetaFile != null) {
    loadCreationMeta(creationMetaFile);
  }

  setTimeInfo(segmentMetadataPropertiesConfiguration);
  _totalDocs = segmentMetadataPropertiesConfiguration.getInt(SEGMENT_TOTAL_DOCS);
}
 

public void init(String configPath) throws ConfigurationException, IOException{
	logger.info("init config");
	
	PropertiesConfiguration config = new PropertiesConfiguration(configPath);
	config.setThrowExceptionOnMissing(true);
	
	this.workDir = config.getString("workDir");
	if (new File(workDir).isDirectory() == false) {
		throw new IllegalArgumentException("workDir is not a directory");
	}
	
	this.hostname = config.getString("hostname", Utils.getHostNameForLinux());
	
	this.jvmContextFile = new File(workDir, "jmxmon.jvm.context.json");
	
	if (jvmContextFile.exists() && jvmContextFile.isFile() && 
			jvmContextFile.length() > 0) {
		logger.info(jvmContextFile.getAbsolutePath() + " is exist, start loading...");
		this.jvmContext = JacksonUtil.readBeanFromFile(jvmContextFile, JVMContext.class);
	} else {
		logger.info(jvmContextFile.getAbsolutePath() + " is not exist");
	}
	
	this.agentPostUrl = config.getString("agent.posturl");
	this.step = config.getInt("step", Constants.defaultStep);
	
	// 默认的jmxHost为localhost，除非通过-D参数设置（线上不建议以远程方式采集，最好每台机器上部署agent，这样agent才能水平伸缩）
	this.jmxHost = System.getProperty("debug.jmx.host");
	if (this.jmxHost == null) {
		this.jmxHost = "localhost";
	}
	
	String[] jmxPortArray = config.getStringArray("jmx.ports");
	jmxPorts = new int[jmxPortArray.length];
	for (int i = 0; i < jmxPortArray.length; i++) {
		jmxPorts[i] = Integer.parseInt(jmxPortArray[i]);
	}
	
	logger.info("init config ok");
}
 

public PinLaterBackendBase(PropertiesConfiguration configuration,
                           String backendName,
                           String serverHostName,
                           long serverStartTimeMillis) {
  this.configuration = Preconditions.checkNotNull(configuration);
  this.backendName = MorePreconditions.checkNotBlank(backendName);
  this.serverHostName = MorePreconditions.checkNotBlank(serverHostName);
  this.serverStartTimeMillis = serverStartTimeMillis;
  this.queryParallelism = configuration.getInt("BACKEND_QUERY_PARALLELISM");
  Preconditions.checkArgument(queryParallelism > 0);
  this.numAutoRetries = configuration.getInt("BACKEND_NUM_AUTO_RETRIES");
  this.numPriorityLevels = configuration.getInt("NUM_PRIORITY_LEVELS");
  Preconditions.checkArgument(numPriorityLevels >= 1);
  this.shardConfigFilePath = System.getProperty("backend_config");
}
 

/**
 * Creates the MySQL shard map from config.
 *
 * @param mysqlConfigStream InputStream containing the MySQL config json.
 * @param configuration  PropertiesConfiguration object.
 * @return A map shardName -> MySQLDataSources.
 */
public static ImmutableMap<String, MySQLDataSources> buildShardMap(
    InputStream mysqlConfigStream,
    PropertiesConfiguration configuration) {
  MySQLConfigSchema mysqlConfig;
  try {
    mysqlConfig = MySQLConfigSchema.read(
        Preconditions.checkNotNull(mysqlConfigStream),
        configuration.getString("SHARD_ALLOWED_HOSTNAME_PREFIX"));
  } catch (Exception e) {
    LOG.error("Failed to load mysql configuration", e);
    throw new RuntimeException(e);
  }

  ImmutableMap.Builder<String, MySQLDataSources> shardMapBuilder =
      new ImmutableMap.Builder<String, MySQLDataSources>();
  int numDbPerQueue = configuration.getInt("MYSQL_NUM_DB_PER_QUEUE", 1);
  for (MySQLConfigSchema.Shard shard : mysqlConfig.shards) {
    MySQLDataSources dataSources = new MySQLDataSources(
        configuration,
        shard.shardConfig.master.host,
        shard.shardConfig.master.port,
        shard.shardConfig.user,
        shard.shardConfig.passwd,
        shard.shardConfig.dequeueOnly);
    // Share data source between databases on the same MySQL instance
    for (int dbId = 0; dbId < numDbPerQueue; dbId++) {
      shardMapBuilder.put(MySQLBackendUtils.constructShardName(shard.id, dbId), dataSources);
    }
  }
  return shardMapBuilder.build();
}
 

public BackendQueueMonitorBase(ImmutableMap<String, T> shardMap,
                               PropertiesConfiguration configuration) {
  this(shardMap,
      configuration.getInt("BACKEND_MONITOR_UPDATE_MAX_SIZE"),
      configuration.getInt("BACKEND_NUM_AUTO_RETRIES"),
      configuration.getInt("MONITOR_LOG_INTERVAL", 1),
      TimeUnit.SECONDS.toMillis(
          configuration.getInt("BACKEND_MONITOR_JOB_CLAIMED_TIMEOUT_SECONDS")),
      TimeUnit.HOURS.toMillis(configuration.getInt(
          "BACKEND_MONITOR_JOB_SUCCEEDED_GC_TTL_HOURS")),
      TimeUnit.HOURS.toMillis(configuration.getInt(
          "BACKEND_MONITOR_JOB_FAILED_GC_TTL_HOURS")),
      configuration.getInt("NUM_PRIORITY_LEVELS"));
}
 

/**
 * For REALTIME consuming segments.
 */
public SegmentMetadataImpl(RealtimeSegmentZKMetadata segmentMetadata, Schema schema) {
  _indexDir = null;
  PropertiesConfiguration segmentMetadataPropertiesConfiguration = new PropertiesConfiguration();
  segmentMetadataPropertiesConfiguration.addProperty(SEGMENT_CREATOR_VERSION, null);
  segmentMetadataPropertiesConfiguration
      .addProperty(SEGMENT_PADDING_CHARACTER, V1Constants.Str.DEFAULT_STRING_PAD_CHAR);
  segmentMetadataPropertiesConfiguration
      .addProperty(SEGMENT_START_TIME, Long.toString(segmentMetadata.getStartTime()));
  segmentMetadataPropertiesConfiguration.addProperty(SEGMENT_END_TIME, Long.toString(segmentMetadata.getEndTime()));
  segmentMetadataPropertiesConfiguration.addProperty(TABLE_NAME, segmentMetadata.getTableName());

  TimeUnit timeUnit = segmentMetadata.getTimeUnit();
  if (timeUnit != null) {
    segmentMetadataPropertiesConfiguration.addProperty(TIME_UNIT, timeUnit.toString());
  } else {
    segmentMetadataPropertiesConfiguration.addProperty(TIME_UNIT, null);
  }

  segmentMetadataPropertiesConfiguration.addProperty(SEGMENT_TOTAL_DOCS, segmentMetadata.getTotalDocs());

  _crc = segmentMetadata.getCrc();
  _creationTime = segmentMetadata.getCreationTime();
  setTimeInfo(segmentMetadataPropertiesConfiguration);
  _columnMetadataMap = null;
  _tableName = segmentMetadata.getTableName();
  _segmentName = segmentMetadata.getSegmentName();
  _allColumns = schema.getColumnNames();
  _schema = schema;
  _totalDocs = segmentMetadataPropertiesConfiguration.getInt(SEGMENT_TOTAL_DOCS);
}
 

/**
 * Creates an instance of the PinLaterRedisBackend.
 *
 * @param configuration configuration parameters for the backend.
 * @param redisConfigStream stream encapsulating the Redis json config.
 * @param serverHostName hostname of the PinLater server.
 * @param serverStartTimeMillis start time of the PinLater server.
 */
public PinLaterRedisBackend(PropertiesConfiguration configuration,
                            InputStream redisConfigStream,
                            String serverHostName,
                            long serverStartTimeMillis) throws Exception {
  super(configuration, "Redis", serverHostName, serverStartTimeMillis);
  this.shardMap = RedisBackendUtils.buildShardMap(redisConfigStream, configuration);
  this.healthChecker = new HealthChecker("PinLaterRedis");
  for (RedisPools redisPools : shardMap.values()) {
    this.healthChecker.addServer(
        redisPools.getHost(),
        redisPools.getPort(),
        new RedisHeartBeater(new JedisClientHelper(), redisPools.getMonitorRedisPool()),
        configuration.getInt("REDIS_HEALTH_CHECK_CONSECUTIVE_FAILURES", 6),
        configuration.getInt("REDIS_HEALTH_CHECK_CONSECUTIVE_SUCCESSES", 6),
        configuration.getInt("REDIS_HEALTH_CHECK_PING_INTERVAL_SECONDS", 5),
        true);  // is live initially
  }

  // Start the JobQueueMonitor scheduled task.
  final int delaySeconds = configuration.getInt("BACKEND_MONITOR_THREAD_DELAY_SECONDS");
  ScheduledExecutorService service = Executors.newSingleThreadScheduledExecutor(
      new ThreadFactoryBuilder().setDaemon(true).setNameFormat("RedisJobQueueMonitor-%d")
          .build());
  service.scheduleWithFixedDelay(
      new RedisQueueMonitor(shardMap, configuration, healthChecker),
      // Randomize initial delay to prevent all servers from running GC at the same time.
      delaySeconds + RANDOM.nextInt(delaySeconds),
      delaySeconds,
      TimeUnit.SECONDS);

  // Load queue names into memory. Silently catch exceptions to avoid failure in initialization.
  // If queue names are not loaded at this time, they will be retried upon requests.
  try {
    reloadQueueNames();
  } catch (Exception e) {
    // Retry the ack.
    Stats.incr("init-queuenames-failure");
    LOG.error("Failed to load queue names upon initialization.", e);
  }

  // Call Base class's initialization function to initialize the futurePool and dequeue
  // semaphoreMap.
  initialize();
}
 

public static ColumnMetadata fromPropertiesConfiguration(String column, PropertiesConfiguration config) {
  Builder builder = new Builder();

  builder.setColumnName(column);
  builder.setCardinality(config.getInt(getKeyFor(column, CARDINALITY)));
  int totalDocs = config.getInt(getKeyFor(column, TOTAL_DOCS));
  builder.setTotalDocs(totalDocs);
  DataType dataType = DataType.valueOf(config.getString(getKeyFor(column, DATA_TYPE)).toUpperCase());
  builder.setDataType(dataType);
  builder.setBitsPerElement(config.getInt(getKeyFor(column, BITS_PER_ELEMENT)));
  builder.setColumnMaxLength(config.getInt(getKeyFor(column, DICTIONARY_ELEMENT_SIZE)));
  builder.setFieldType(FieldType.valueOf(config.getString(getKeyFor(column, COLUMN_TYPE)).toUpperCase()));
  builder.setIsSorted(config.getBoolean(getKeyFor(column, IS_SORTED)));
  builder.setContainsNulls(config.getBoolean(getKeyFor(column, HAS_NULL_VALUE)));
  builder.setHasDictionary(config.getBoolean(getKeyFor(column, HAS_DICTIONARY), true));
  builder.setHasInvertedIndex(config.getBoolean(getKeyFor(column, HAS_INVERTED_INDEX)));
  builder.setSingleValue(config.getBoolean(getKeyFor(column, IS_SINGLE_VALUED)));
  builder.setMaxNumberOfMultiValues(config.getInt(getKeyFor(column, MAX_MULTI_VALUE_ELEMTS)));
  builder.setTotalNumberOfEntries(config.getInt(getKeyFor(column, TOTAL_NUMBER_OF_ENTRIES)));
  builder.setAutoGenerated(config.getBoolean(getKeyFor(column, IS_AUTO_GENERATED), false));
  builder.setDefaultNullValueString(config.getString(getKeyFor(column, DEFAULT_NULL_VALUE), null));
  builder.setTimeUnit(TimeUnit.valueOf(config.getString(TIME_UNIT, "DAYS").toUpperCase()));
  builder.setTextIndexType(config.getString(getKeyFor(column, TEXT_INDEX_TYPE), TextIndexType.NONE.name()));

  char paddingCharacter = V1Constants.Str.LEGACY_STRING_PAD_CHAR;
  if (config.containsKey(SEGMENT_PADDING_CHARACTER)) {
    String padding = config.getString(SEGMENT_PADDING_CHARACTER);
    paddingCharacter = StringEscapeUtils.unescapeJava(padding).charAt(0);
  }
  builder.setPaddingCharacter(paddingCharacter);

  String dateTimeFormat = config.getString(getKeyFor(column, DATETIME_FORMAT), null);
  if (dateTimeFormat != null) {
    builder.setDateTimeFormat(dateTimeFormat);
  }

  String dateTimeGranularity = config.getString(getKeyFor(column, DATETIME_GRANULARITY), null);
  if (dateTimeGranularity != null) {
    builder.setDateTimeGranularity(dateTimeGranularity);
  }

  // Set min/max value if available.
  String minString = config.getString(getKeyFor(column, MIN_VALUE), null);
  String maxString = config.getString(getKeyFor(column, MAX_VALUE), null);
  if ((minString != null) && (maxString != null)) {
    switch (dataType) {
      case INT:
        builder.setMinValue(Integer.valueOf(minString));
        builder.setMaxValue(Integer.valueOf(maxString));
        break;
      case LONG:
        builder.setMinValue(Long.valueOf(minString));
        builder.setMaxValue(Long.valueOf(maxString));
        break;
      case FLOAT:
        builder.setMinValue(Float.valueOf(minString));
        builder.setMaxValue(Float.valueOf(maxString));
        break;
      case DOUBLE:
        builder.setMinValue(Double.valueOf(minString));
        builder.setMaxValue(Double.valueOf(maxString));
        break;
      case STRING:
        builder.setMinValue(minString);
        builder.setMaxValue(maxString);
        break;
      default:
        throw new IllegalStateException("Unsupported data type: " + dataType + " for column: " + column);
    }
  }

  String partitionFunctionName =
      config.getString(getKeyFor(column, V1Constants.MetadataKeys.Column.PARTITION_FUNCTION));
  if (partitionFunctionName != null) {
    int numPartitions = config.getInt(getKeyFor(column, V1Constants.MetadataKeys.Column.NUM_PARTITIONS));
    PartitionFunction partitionFunction =
        PartitionFunctionFactory.getPartitionFunction(partitionFunctionName, numPartitions);
    builder.setPartitionFunction(partitionFunction);
    builder.setNumPartitions(numPartitions);
    builder.setPartitions(ColumnPartitionMetadata
        .extractPartitions(config.getList(getKeyFor(column, V1Constants.MetadataKeys.Column.PARTITION_VALUES))));
  }

  return builder.build();
}
 

private void init(PropertiesConfiguration segmentMetadataPropertiesConfiguration) {
  if (segmentMetadataPropertiesConfiguration.containsKey(SEGMENT_CREATOR_VERSION)) {
    _creatorName = segmentMetadataPropertiesConfiguration.getString(SEGMENT_CREATOR_VERSION);
  }

  if (segmentMetadataPropertiesConfiguration.containsKey(SEGMENT_PADDING_CHARACTER)) {
    String padding = segmentMetadataPropertiesConfiguration.getString(SEGMENT_PADDING_CHARACTER);
    _paddingCharacter = StringEscapeUtils.unescapeJava(padding).charAt(0);
  }

  String versionString =
      segmentMetadataPropertiesConfiguration.getString(SEGMENT_VERSION, SegmentVersion.v1.toString());
  _segmentVersion = SegmentVersion.valueOf(versionString);

  // NOTE: here we only add physical columns as virtual columns should not be loaded from metadata file
  // NOTE: getList() will always return an non-null List with trimmed strings:
  // - If key does not exist, it will return an empty list
  // - If key exists but value is missing, it will return a singleton list with an empty string
  addPhysicalColumns(segmentMetadataPropertiesConfiguration.getList(DIMENSIONS), _allColumns);
  addPhysicalColumns(segmentMetadataPropertiesConfiguration.getList(METRICS), _allColumns);
  addPhysicalColumns(segmentMetadataPropertiesConfiguration.getList(TIME_COLUMN_NAME), _allColumns);
  addPhysicalColumns(segmentMetadataPropertiesConfiguration.getList(DATETIME_COLUMNS), _allColumns);

  //set the table name
  _tableName = segmentMetadataPropertiesConfiguration.getString(TABLE_NAME);

  // Set segment name.
  _segmentName = segmentMetadataPropertiesConfiguration.getString(SEGMENT_NAME);

  // Build column metadata map, schema and hll derived column map.
  for (String column : _allColumns) {
    ColumnMetadata columnMetadata =
        ColumnMetadata.fromPropertiesConfiguration(column, segmentMetadataPropertiesConfiguration);
    _columnMetadataMap.put(column, columnMetadata);
    _schema.addField(columnMetadata.getFieldSpec());
  }

  // Build star-tree v2 metadata
  int starTreeV2Count =
      segmentMetadataPropertiesConfiguration.getInt(StarTreeV2Constants.MetadataKey.STAR_TREE_COUNT, 0);
  if (starTreeV2Count > 0) {
    _starTreeV2MetadataList = new ArrayList<>(starTreeV2Count);
    for (int i = 0; i < starTreeV2Count; i++) {
      _starTreeV2MetadataList.add(new StarTreeV2Metadata(
          segmentMetadataPropertiesConfiguration.subset(StarTreeV2Constants.MetadataKey.getStarTreePrefix(i))));
    }
  }
}