Java Code Examples for org.apache.hadoop.conf.Configuration#getDouble()

public DefaultSpeculator
    (Configuration conf, AppContext context,
     TaskRuntimeEstimator estimator, Clock clock) {
  super(DefaultSpeculator.class.getName());

  this.conf = conf;
  this.context = context;
  this.estimator = estimator;
  this.clock = clock;
  this.eventHandler = context.getEventHandler();
  this.soonestRetryAfterNoSpeculate =
      conf.getLong(MRJobConfig.SPECULATIVE_RETRY_AFTER_NO_SPECULATE,
              MRJobConfig.DEFAULT_SPECULATIVE_RETRY_AFTER_NO_SPECULATE);
  this.soonestRetryAfterSpeculate =
      conf.getLong(MRJobConfig.SPECULATIVE_RETRY_AFTER_SPECULATE,
              MRJobConfig.DEFAULT_SPECULATIVE_RETRY_AFTER_SPECULATE);
  this.proportionRunningTasksSpeculatable =
      conf.getDouble(MRJobConfig.SPECULATIVECAP_RUNNING_TASKS,
              MRJobConfig.DEFAULT_SPECULATIVECAP_RUNNING_TASKS);
  this.proportionTotalTasksSpeculatable =
      conf.getDouble(MRJobConfig.SPECULATIVECAP_TOTAL_TASKS,
              MRJobConfig.DEFAULT_SPECULATIVECAP_TOTAL_TASKS);
  this.minimumAllowedSpeculativeTasks =
      conf.getInt(MRJobConfig.SPECULATIVE_MINIMUM_ALLOWED_TASKS,
              MRJobConfig.DEFAULT_SPECULATIVE_MINIMUM_ALLOWED_TASKS);
}
 

@Override
protected void setup(Context context) throws IOException, InterruptedException {
    Configuration configuration = context.getConfiguration();

    verbose = configuration.getBoolean(LouvainMR.VERBOSE,false);
    nb_pass = configuration.getInt(LouvainMR.NB_PASS, 0);
    precision = configuration.getDouble(LouvainMR.PRECISION, 0.000001);
    display_level = configuration.getInt(LouvainMR.DISPLAY_LEVEL, -1);
    outpath = configuration.get(LouvainMR.OUT_PATH);

    System.out.println("verbose = " + verbose);
    System.out.println("display_level = " + display_level);
    System.out.println("outpath = " + outpath);


    super.setup(context);

}
 

public LegacySpeculator
    (Configuration conf, TaskRuntimeEstimator estimator, Clock clock, Vertex vertex) {
  super(LegacySpeculator.class.getName());
  this.vertex = vertex;
  this.estimator = estimator;
  this.clock = clock;
  taskTimeout = conf.getLong(
          TezConfiguration.TEZ_AM_LEGACY_SPECULATIVE_SINGLE_TASK_VERTEX_TIMEOUT,
          TezConfiguration.TEZ_AM_LEGACY_SPECULATIVE_SINGLE_TASK_VERTEX_TIMEOUT_DEFAULT);
  soonestRetryAfterNoSpeculate = conf.getLong(
          TezConfiguration.TEZ_AM_SOONEST_RETRY_AFTER_NO_SPECULATE,
          TezConfiguration.TEZ_AM_SOONEST_RETRY_AFTER_NO_SPECULATE_DEFAULT);
  soonestRetryAfterSpeculate = conf.getLong(
          TezConfiguration.TEZ_AM_SOONEST_RETRY_AFTER_SPECULATE,
          TezConfiguration.TEZ_AM_SOONEST_RETRY_AFTER_SPECULATE_DEFAULT);
  proportionRunningTasksSpeculatable = conf.getDouble(
          TezConfiguration.TEZ_AM_PROPORTION_RUNNING_TASKS_SPECULATABLE,
          TezConfiguration.TEZ_AM_PROPORTION_RUNNING_TASKS_SPECULATABLE_DEFAULT);
  proportionTotalTasksSpeculatable = conf.getDouble(
          TezConfiguration.TEZ_AM_PROPORTION_TOTAL_TASKS_SPECULATABLE,
          TezConfiguration.TEZ_AM_PROPORTION_TOTAL_TASKS_SPECULATABLE_DEFAULT);
  minimumAllowedSpeculativeTasks = conf.getInt(
          TezConfiguration.TEZ_AM_MINIMUM_ALLOWED_SPECULATIVE_TASKS,
          TezConfiguration.TEZ_AM_MINIMUM_ALLOWED_SPECULATIVE_TASKS_DEFAULT);
}
 

@Override
protected void configureForRegion(HRegion region) {
  super.configureForRegion(region);
  Configuration conf = getConf();
  TableDescriptor desc = region.getTableDescriptor();
  if (desc != null) {
    this.desiredMaxFileSize = desc.getMaxFileSize();
  }
  if (this.desiredMaxFileSize <= 0) {
    this.desiredMaxFileSize = conf.getLong(HConstants.HREGION_MAX_FILESIZE,
      HConstants.DEFAULT_MAX_FILE_SIZE);
  }
  double jitter = conf.getDouble("hbase.hregion.max.filesize.jitter", 0.25D);
  this.jitterRate = (RANDOM.nextFloat() - 0.5D) * jitter;
  long jitterValue = (long) (this.desiredMaxFileSize * this.jitterRate);
  // make sure the long value won't overflow with jitter
  if (this.jitterRate > 0 && jitterValue > (Long.MAX_VALUE - this.desiredMaxFileSize)) {
    this.desiredMaxFileSize = Long.MAX_VALUE;
  } else {
    this.desiredMaxFileSize += jitterValue;
  }
}
 

private void initInmemoryFlushSize(Configuration conf) {
  double factor = 0;
  long memstoreFlushSize = getRegionServices().getMemStoreFlushSize();
  int numStores = getRegionServices().getNumStores();
  if (numStores <= 1) {
    // Family number might also be zero in some of our unit test case
    numStores = 1;
  }
  factor = conf.getDouble(IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, 0.0);
  if(factor != 0.0) {
    // multiply by a factor (the same factor for all index types)
    inmemoryFlushSize = (long) (factor * memstoreFlushSize) / numStores;
  } else {
    inmemoryFlushSize = IN_MEMORY_FLUSH_MULTIPLIER *
        conf.getLong(MemStoreLAB.CHUNK_SIZE_KEY, MemStoreLAB.CHUNK_SIZE_DEFAULT);
    inmemoryFlushSize -= ChunkCreator.SIZEOF_CHUNK_HEADER;
  }
}
 

public DefaultSpeculator
    (Configuration conf, AppContext context,
     TaskRuntimeEstimator estimator, Clock clock) {
  super(DefaultSpeculator.class.getName());

  this.conf = conf;
  this.context = context;
  this.estimator = estimator;
  this.clock = clock;
  this.eventHandler = context.getEventHandler();
  this.soonestRetryAfterNoSpeculate =
      conf.getLong(MRJobConfig.SPECULATIVE_RETRY_AFTER_NO_SPECULATE,
              MRJobConfig.DEFAULT_SPECULATIVE_RETRY_AFTER_NO_SPECULATE);
  this.soonestRetryAfterSpeculate =
      conf.getLong(MRJobConfig.SPECULATIVE_RETRY_AFTER_SPECULATE,
              MRJobConfig.DEFAULT_SPECULATIVE_RETRY_AFTER_SPECULATE);
  this.proportionRunningTasksSpeculatable =
      conf.getDouble(MRJobConfig.SPECULATIVECAP_RUNNING_TASKS,
              MRJobConfig.DEFAULT_SPECULATIVECAP_RUNNING_TASKS);
  this.proportionTotalTasksSpeculatable =
      conf.getDouble(MRJobConfig.SPECULATIVECAP_TOTAL_TASKS,
              MRJobConfig.DEFAULT_SPECULATIVECAP_TOTAL_TASKS);
  this.minimumAllowedSpeculativeTasks =
      conf.getInt(MRJobConfig.SPECULATIVE_MINIMUM_ALLOWED_TASKS,
              MRJobConfig.DEFAULT_SPECULATIVE_MINIMUM_ALLOWED_TASKS);
}
 

public TimelineMetricSplitPointComputer(Configuration metricsConf,
                                        Configuration hbaseConf,
                                        TimelineMetricMetadataManager timelineMetricMetadataManager) {

  String componentsString = metricsConf.get(TIMELINE_METRIC_INITIAL_CONFIGURED_MASTER_COMPONENTS, "");
  if (StringUtils.isNotEmpty(componentsString)) {
    masterComponents.addAll(Arrays.asList(componentsString.split(",")));
  }

 componentsString = metricsConf.get(TIMELINE_METRIC_INITIAL_CONFIGURED_SLAVE_COMPONENTS, "");
  if (StringUtils.isNotEmpty(componentsString)) {
    slaveComponents.addAll(Arrays.asList(componentsString.split(",")));
  }

  this.timelineMetricMetadataManager = timelineMetricMetadataManager;
  hbaseTotalHeapsize = metricsConf.getDouble("hbase_total_heapsize", 1024*1024*1024);
  hbaseMemstoreUpperLimit = hbaseConf.getDouble("hbase.regionserver.global.memstore.upperLimit", 0.3);
  hbaseMemstoreFlushSize = hbaseConf.getDouble("hbase.hregion.memstore.flush.size", 134217728);
}
 

@Override
public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
    OrcNewSplit orcNewSplit = (OrcNewSplit) inputSplit;
    Configuration configuration = taskAttemptContext.getConfiguration();
    double maxMergeDistance = configuration.getDouble(MAX_MERGE_DISTANCE,MAX_MERGE_DISTANCE_DEFAULT);
    double maxReadSize = configuration.getDouble(MAX_READ_SIZE,MAX_READ_SIZE_DEFAULT);
    double streamBufferSize = configuration.getDouble(STREAM_BUFFER_SIZE,STREAM_BUFFER_SIZE_DEFAULT);
    Path path = orcNewSplit.getPath();
    FileSystem fileSystem = FileSystem.get(path.toUri(),configuration);
    long size = fileSystem.getFileStatus(path).getLen();
    FSDataInputStream inputStream = fileSystem.open(path);
    rowStruct = getRowStruct(configuration);
    predicate = getSplicePredicate(configuration);
    List<Integer> partitions = getPartitionIds(configuration);
    List<Integer> columnIds = getColumnIds(configuration);



    List<String> values = null;
    try {
        values = Warehouse.getPartValuesFromPartName(((OrcNewSplit) inputSplit).getPath().toString());
    } catch (MetaException me) {
        throw new IOException(me);
    }
    OrcDataSource orcDataSource = new HdfsOrcDataSource(path.toString(), size, new DataSize(maxMergeDistance, DataSize.Unit.MEGABYTE),
            new DataSize(maxReadSize, DataSize.Unit.MEGABYTE),
            new DataSize(streamBufferSize, DataSize.Unit.MEGABYTE), inputStream);
    OrcReader orcReader = new OrcReader(orcDataSource, new OrcMetadataReader(), new DataSize(maxMergeDistance, DataSize.Unit.MEGABYTE),
            new DataSize(maxReadSize, DataSize.Unit.MEGABYTE));
    orcRecordReader =
        orcReader.createRecordReader(getColumnsAndTypes(columnIds, rowStruct),
                                     predicate, orcNewSplit.getStart(), orcNewSplit.getLength(),
                                     HIVE_STORAGE_TIME_ZONE, new AggregatedMemoryContext(),
                                     partitions, values);

}
 

private static double parseDecayFactor(String ns, Configuration conf) {
  double factor = conf.getDouble(ns + "." +
      IPC_CALLQUEUE_DECAYSCHEDULER_FACTOR_KEY,
    IPC_CALLQUEUE_DECAYSCHEDULER_FACTOR_DEFAULT
  );

  if (factor <= 0 || factor >= 1) {
    throw new IllegalArgumentException("Decay Factor " +
      "must be between 0 and 1");
  }

  return factor;
}
 

public AdaptiveMemStoreCompactionStrategy(Configuration conf, String cfName) {
  super(conf, cfName);
  compactionThreshold = conf.getDouble(ADAPTIVE_COMPACTION_THRESHOLD_KEY,
      ADAPTIVE_COMPACTION_THRESHOLD_DEFAULT);
  initialCompactionProbability = conf.getDouble(ADAPTIVE_INITIAL_COMPACTION_PROBABILITY_KEY,
      ADAPTIVE_INITIAL_COMPACTION_PROBABILITY_DEFAULT);
  resetStats();
}
 

private static double parseDecayFactor(String ns, Configuration conf) {
  double factor = conf.getDouble(ns + "." +
      IPC_CALLQUEUE_DECAYSCHEDULER_FACTOR_KEY,
    IPC_CALLQUEUE_DECAYSCHEDULER_FACTOR_DEFAULT
  );

  if (factor <= 0 || factor >= 1) {
    throw new IllegalArgumentException("Decay Factor " +
      "must be between 0 and 1");
  }

  return factor;
}
 

public void onConfigurationChange(Configuration conf) {
  // update CoDel Scheduler tunables
  int codelTargetDelay = conf.getInt(CALL_QUEUE_CODEL_TARGET_DELAY,
    CALL_QUEUE_CODEL_DEFAULT_TARGET_DELAY);
  int codelInterval = conf.getInt(CALL_QUEUE_CODEL_INTERVAL, CALL_QUEUE_CODEL_DEFAULT_INTERVAL);
  double codelLifoThreshold = conf.getDouble(CALL_QUEUE_CODEL_LIFO_THRESHOLD,
    CALL_QUEUE_CODEL_DEFAULT_LIFO_THRESHOLD);

  for (BlockingQueue<CallRunner> queue : queues) {
    if (queue instanceof AdaptiveLifoCoDelCallQueue) {
      ((AdaptiveLifoCoDelCallQueue) queue).updateTunables(codelTargetDelay, codelInterval,
        codelLifoThreshold);
    }
  }
}
 

protected void setup(Context context) throws IOException,
		InterruptedException {
	Configuration conf = context.getConfiguration();
	regularizationFactor = conf.getDouble(
			"lr.iteration.regulariztion.factor",
			DEFAULT_REGULARIZATION_FACTOR);
	lbfgs = new QNMinimizer();
	lbfgs.setRobustOptions();
}
 

@Override
protected void setup(Context context) throws IOException, InterruptedException {
    Configuration configuration = context.getConfiguration();
    verbose = configuration.getBoolean(LouvainMR.VERBOSE, false);
    precision = configuration.getDouble(LouvainMR.PRECISION, 0.000001);
    display_level = configuration.getInt(LouvainMR.DISPLAY_LEVEL, -1);
    this.outpath = configuration.get(LouvainMR.OUT_PATH);
    System.out.println("verbose = " + verbose);
    System.out.println("display_level = " + display_level);
    System.out.println("outpath = " + outpath);

    super.setup(context);
}
 

/**
 * Initializes the HMaster. The steps are as follows:
 * <p>
 * <ol>
 * <li>Initialize the local HRegionServer
 * <li>Start the ActiveMasterManager.
 * </ol>
 * <p>
 * Remaining steps of initialization occur in
 * {@link #finishActiveMasterInitialization(MonitoredTask)} after the master becomes the
 * active one.
 */
public HMaster(final Configuration conf) throws IOException {
  super(conf);
  TraceUtil.initTracer(conf);
  try {
    if (conf.getBoolean(MAINTENANCE_MODE, false)) {
      LOG.info("Detected {}=true via configuration.", MAINTENANCE_MODE);
      maintenanceMode = true;
    } else if (Boolean.getBoolean(MAINTENANCE_MODE)) {
      LOG.info("Detected {}=true via environment variables.", MAINTENANCE_MODE);
      maintenanceMode = true;
    } else {
      maintenanceMode = false;
    }
    this.rsFatals = new MemoryBoundedLogMessageBuffer(
        conf.getLong("hbase.master.buffer.for.rs.fatals", 1 * 1024 * 1024));
    LOG.info("hbase.rootdir={}, hbase.cluster.distributed={}", getDataRootDir(),
        this.conf.getBoolean(HConstants.CLUSTER_DISTRIBUTED, false));

    // Disable usage of meta replicas in the master
    this.conf.setBoolean(HConstants.USE_META_REPLICAS, false);

    decorateMasterConfiguration(this.conf);

    // Hack! Maps DFSClient => Master for logs.  HDFS made this
    // config param for task trackers, but we can piggyback off of it.
    if (this.conf.get("mapreduce.task.attempt.id") == null) {
      this.conf.set("mapreduce.task.attempt.id", "hb_m_" + this.serverName.toString());
    }

    this.metricsMaster = new MetricsMaster(new MetricsMasterWrapperImpl(this));

    // preload table descriptor at startup
    this.preLoadTableDescriptors = conf.getBoolean("hbase.master.preload.tabledescriptors", true);

    this.maxBalancingTime = getMaxBalancingTime();
    this.maxRitPercent = conf.getDouble(HConstants.HBASE_MASTER_BALANCER_MAX_RIT_PERCENT,
        HConstants.DEFAULT_HBASE_MASTER_BALANCER_MAX_RIT_PERCENT);

    // Do we publish the status?

    boolean shouldPublish = conf.getBoolean(HConstants.STATUS_PUBLISHED,
        HConstants.STATUS_PUBLISHED_DEFAULT);
    Class<? extends ClusterStatusPublisher.Publisher> publisherClass =
        conf.getClass(ClusterStatusPublisher.STATUS_PUBLISHER_CLASS,
            ClusterStatusPublisher.DEFAULT_STATUS_PUBLISHER_CLASS,
            ClusterStatusPublisher.Publisher.class);

    if (shouldPublish) {
      if (publisherClass == null) {
        LOG.warn(HConstants.STATUS_PUBLISHED + " is true, but " +
            ClusterStatusPublisher.DEFAULT_STATUS_PUBLISHER_CLASS +
            " is not set - not publishing status");
      } else {
        clusterStatusPublisherChore = new ClusterStatusPublisher(this, conf, publisherClass);
        LOG.debug("Created {}", this.clusterStatusPublisherChore);
        getChoreService().scheduleChore(clusterStatusPublisherChore);
      }
    }

    // Some unit tests don't need a cluster, so no zookeeper at all
    if (!conf.getBoolean("hbase.testing.nocluster", false)) {
      this.metaRegionLocationCache = new MetaRegionLocationCache(this.zooKeeper);
      this.activeMasterManager = createActiveMasterManager(zooKeeper, serverName, this);
    } else {
      this.metaRegionLocationCache = null;
      this.activeMasterManager = null;
    }
    cachedClusterId = new CachedClusterId(this, conf);
  } catch (Throwable t) {
    // Make sure we log the exception. HMaster is often started via reflection and the
    // cause of failed startup is lost.
    LOG.error("Failed construction of Master", t);
    throw t;
  }
}
 

public LossyCounting(String name, Configuration conf, LossyCountingListener<T> listener) {
  this(name, conf.getDouble(HConstants.DEFAULT_LOSSY_COUNTING_ERROR_RATE, 0.02), listener);
}
 

public FaultyMobStoreCompactor(Configuration conf, HStore store) {
  super(conf, store);
  failureProb = conf.getDouble("hbase.mob.compaction.fault.probability", 0.1);
}
 

public static double getLookupRatio(Configuration configuration) {
  return configuration.getDouble(BLUR_LOOKUP_RATIO_PER_SHARD, DEFAULT_LOOKUP_RATIO);
}
 

/**
 * Extracts the percent of Regions for a table to have been reported to enable quota violation
 * state change.
 *
 * @param conf The configuration object.
 * @return The percent of regions reported to use.
 */
static Double getRegionReportPercent(Configuration conf) {
  return conf.getDouble(QUOTA_OBSERVER_CHORE_REPORT_PERCENT_KEY,
      QUOTA_OBSERVER_CHORE_REPORT_PERCENT_DEFAULT);
}
 

/**
 * Extracts the percent of Regions for a table to have been reported to enable quota violation
 * state change.
 *
 * @param conf The configuration object.
 * @return The percent of regions reported to use.
 */
static Double getRegionReportPercent(Configuration conf) {
  return conf.getDouble(POLICY_REFRESHER_CHORE_REPORT_PERCENT_KEY,
      POLICY_REFRESHER_CHORE_REPORT_PERCENT_DEFAULT);
}