Java Code Examples for org.apache.hadoop.conf.Configuration#setDouble()

public static void set(final Configuration conf, final String name, final Object value) {
    if (null == value) {
        return;
    }
    
    if (value instanceof String) {
        conf.set(name, (String) value);
    } else if (value instanceof Integer) {
        conf.setInt(name, ((Integer) value).intValue());
    } else if (value instanceof Long) {
        conf.setLong(name, ((Long) value).longValue());
    } else if (value instanceof Float) {
        conf.setFloat(name, ((Float) value).floatValue());
    } else if (value instanceof Double) {
        conf.setDouble(name, ((Double) value).doubleValue());
    } else if (value instanceof Boolean) {
        conf.setBoolean(name, ((Boolean) value).booleanValue());
    } else {
        throw (new IllegalArgumentException(value.getClass().getSimpleName() + " is unhandled."));
    }
}
 

private void setConfiguration(Configuration conf, String property, Object value)
{
  if (value instanceof Integer) {
    conf.setInt(property, (Integer)value);
  } else if (value instanceof Boolean) {
    conf.setBoolean(property, (Boolean)value);
  } else if (value instanceof Long) {
    conf.setLong(property, (Long)value);
  } else if (value instanceof Float) {
    conf.setFloat(property, (Float)value);
  } else if (value instanceof Double) {
    conf.setDouble(property, (Double)value);
  } else {
    conf.set(property, value.toString());
  }
}
 

private void setConfiguration(Configuration conf, String property, Object value)
{
  if (value instanceof Integer) {
    conf.setInt(property, (Integer)value);
  } else if (value instanceof Boolean) {
    conf.setBoolean(property, (Boolean)value);
  } else if (value instanceof Long) {
    conf.setLong(property, (Long)value);
  } else if (value instanceof Float) {
    conf.setFloat(property, (Float)value);
  } else if (value instanceof Double) {
    conf.setDouble(property, (Double)value);
  } else {
    conf.set(property, value.toString());
  }
}
 

/**
 * Note that all subclasses of this class must provide a public constructor
 * that has the exact same list of arguments.
 */
TestBase(final Configuration conf, final TestOptions options, final Status status) {
  this.conf = conf;
  this.receiverHost = this.conf == null? null: SpanReceiverHost.getInstance(conf);
  this.opts = options;
  this.status = status;
  this.testName = this.getClass().getSimpleName();
  if (options.traceRate >= 1.0) {
    this.traceSampler = Sampler.ALWAYS;
  } else if (options.traceRate > 0.0) {
    conf.setDouble("hbase.sampler.fraction", options.traceRate);
    this.traceSampler = new ProbabilitySampler(new HBaseHTraceConfiguration(conf));
  } else {
    this.traceSampler = Sampler.NEVER;
  }
  everyN = (int) (opts.totalRows / (opts.totalRows * opts.sampleRate));
  if (options.isValueZipf()) {
    this.zipf = new RandomDistribution.Zipf(this.rand, 1, options.getValueSize(), 1.2);
  }
  LOG.info("Sampling 1 every " + everyN + " out of " + opts.perClientRunRows + " total rows.");
}
 

private static void initConf(Configuration conf) {

    conf.setInt("hfile.format.version", 3);
    conf.setLong(TimeToLiveHFileCleaner.TTL_CONF_KEY, 0);
    conf.setInt("hbase.client.retries.number", 100);
    conf.setInt("hbase.hregion.max.filesize", 200000000);
    conf.setInt("hbase.hregion.memstore.flush.size", 800000);
    conf.setInt("hbase.hstore.blockingStoreFiles", 150);
    conf.setInt("hbase.hstore.compaction.throughput.lower.bound", 52428800);
    conf.setInt("hbase.hstore.compaction.throughput.higher.bound", 2 * 52428800);
    conf.setDouble("hbase.mob.compaction.fault.probability", failureProb);
    conf.set(MobStoreEngine.MOB_COMPACTOR_CLASS_KEY,
      FaultyMobStoreCompactor.class.getName());
    conf.setBoolean("hbase.table.sanity.checks", false);
    conf.setLong(MobConstants.MIN_AGE_TO_ARCHIVE_KEY, 20000);

  }
 

@Test(timeout = 5000)
public void testReserveFractionConfigured() throws TezException {
  Configuration conf = new Configuration(this.conf);
  conf.setDouble(TezConfiguration.TEZ_TASK_SCALE_MEMORY_RESERVE_FRACTION, 0.5d);
  MemoryDistributor dist = new MemoryDistributor(2, 1, conf);
  
  dist.setJvmMemory(10000l);

  // First request
  MemoryUpdateCallbackForTest e1Callback = new MemoryUpdateCallbackForTest();
  InputContext e1InputContext1 = createTestInputContext();
  InputDescriptor e1InDesc1 = createTestInputDescriptor();
  dist.requestMemory(10000, e1Callback, e1InputContext1, e1InDesc1);
  
  // Second request
  MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest();
  InputContext e2InputContext2 = createTestInputContext();
  InputDescriptor e2InDesc2 = createTestInputDescriptor();
  dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2);
  
  // Third request - output
  MemoryUpdateCallbackForTest e3Callback = new MemoryUpdateCallbackForTest();
  OutputContext e3OutputContext1 = createTestOutputContext();
  OutputDescriptor e3OutDesc2 = createTestOutputDescriptor();
  dist.requestMemory(5000, e3Callback, e3OutputContext1, e3OutDesc2);
  
  dist.makeInitialAllocations();
  
  // Total available: 50% of 10K = 7000
  // 3 requests - 10K, 10K, 5K
  // Scale down to - 2000, 2000, 1000
  assertEquals(2000, e1Callback.assigned);
  assertEquals(2000, e2Callback.assigned);
  assertEquals(1000, e3Callback.assigned);
}
 

/**
 * Sets default conf.
 */
@Before
public void setDefaultConf() {
  try {
    defaultConf = new Configuration(false);
    defaultConf.set("fs.defaultFS", "file:///");
    defaultConf.setBoolean(TezConfiguration.TEZ_LOCAL_MODE, true);
    defaultConf.setBoolean(TezConfiguration.TEZ_AM_SPECULATION_ENABLED, true);
    defaultConf.setFloat(
        ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MIN_SRC_FRACTION, 1);
    defaultConf.setFloat(
        ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION, 1);
    localFs = FileSystem.getLocal(defaultConf);
    String stagingDir =
        "target" + Path.SEPARATOR + TestSpeculation.class.getName()
            + "-tmpDir";
    defaultConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDir);
    defaultConf.setClass(TezConfiguration.TEZ_AM_TASK_ESTIMATOR_CLASS,
        estimatorClass,
        TaskRuntimeEstimator.class);
    defaultConf.setInt(TezConfiguration.TEZ_AM_MINIMUM_ALLOWED_SPECULATIVE_TASKS, 20);
    defaultConf.setDouble(TezConfiguration.TEZ_AM_PROPORTION_TOTAL_TASKS_SPECULATABLE, 0.2);
    defaultConf.setDouble(TezConfiguration.TEZ_AM_PROPORTION_RUNNING_TASKS_SPECULATABLE, 0.25);
    defaultConf.setLong(TezConfiguration.TEZ_AM_SOONEST_RETRY_AFTER_NO_SPECULATE, 25);
    defaultConf.setLong(TezConfiguration.TEZ_AM_SOONEST_RETRY_AFTER_SPECULATE, 50);
    defaultConf.setInt(TezConfiguration.TEZ_AM_ESTIMATOR_EXPONENTIAL_SKIP_INITIALS, 2);
  } catch (IOException e) {
    throw new RuntimeException("init failure", e);
  }
}
 

private Configuration getConf() {
  Configuration conf = new Configuration(false);
  conf.set("test1", "value1");
  conf.setBoolean("test2", true);
  conf.setDouble("test3", 1.2345);
  conf.setInt("test4", 34567);
  conf.setLong("test5", 1234567890L);
  conf.setStrings("test6", "S1", "S2", "S3");
  return conf;
}
 

@Test(timeout = 5000)
public void testReserveFractionConfigured() {
  Configuration conf = new Configuration(this.conf);
  conf.setDouble(TezJobConfig.TEZ_RUNTIME_SCALE_TASK_MEMORY_RESERVE_FRACTION, 0.5d);
  MemoryDistributor dist = new MemoryDistributor(2, 1, conf);
  
  dist.setJvmMemory(10000l);

  // First request
  MemoryUpdateCallbackForTest e1Callback = new MemoryUpdateCallbackForTest();
  TezInputContext e1InputContext1 = createTestInputContext();
  InputDescriptor e1InDesc1 = createTestInputDescriptor();
  dist.requestMemory(10000, e1Callback, e1InputContext1, e1InDesc1);
  
  // Second request
  MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest();
  TezInputContext e2InputContext2 = createTestInputContext();
  InputDescriptor e2InDesc2 = createTestInputDescriptor();
  dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2);
  
  // Third request - output
  MemoryUpdateCallbackForTest e3Callback = new MemoryUpdateCallbackForTest();
  TezOutputContext e3OutputContext1 = createTestOutputContext();
  OutputDescriptor e3OutDesc2 = createTestOutputDescriptor();
  dist.requestMemory(5000, e3Callback, e3OutputContext1, e3OutDesc2);
  
  dist.makeInitialAllocations();
  
  // Total available: 50% of 10K = 7000
  // 3 requests - 10K, 10K, 5K
  // Scale down to - 2000, 2000, 1000
  assertEquals(2000, e1Callback.assigned);
  assertEquals(2000, e2Callback.assigned);
  assertEquals(1000, e3Callback.assigned);
}
 

private Configuration getConf() {
  Configuration conf = new Configuration(false);
  conf.set("test1", "value1");
  conf.setBoolean("test2", true);
  conf.setDouble("test3", 1.2345);
  conf.setInt("test4", 34567);
  conf.setLong("test5", 1234567890L);
  conf.setStrings("test6", "S1", "S2", "S3");
  return conf;
}
 

private static final void setParameter(
    final Configuration config,
    final Class<?> scope,
    final Object val,
    final ParameterEnum configItem) {
  if (val != null) {
    if (val instanceof Long) {
      config.setLong(
          GeoWaveConfiguratorBase.enumToConfKey(scope, configItem.self()),
          ((Long) val));
    } else if (val instanceof Double) {
      config.setDouble(
          GeoWaveConfiguratorBase.enumToConfKey(scope, configItem.self()),
          ((Double) val));
    } else if (val instanceof Boolean) {
      config.setBoolean(
          GeoWaveConfiguratorBase.enumToConfKey(scope, configItem.self()),
          ((Boolean) val));
    } else if (val instanceof Integer) {
      config.setInt(
          GeoWaveConfiguratorBase.enumToConfKey(scope, configItem.self()),
          ((Integer) val));
    } else if (val instanceof Class) {
      config.setClass(
          GeoWaveConfiguratorBase.enumToConfKey(scope, configItem.self()),
          ((Class) val),
          ((Class) val));
    } else if (val instanceof byte[]) {
      config.set(
          GeoWaveConfiguratorBase.enumToConfKey(scope, configItem.self()),
          ByteArrayUtils.byteArrayToString((byte[]) val));
    } else {
      config.set(GeoWaveConfiguratorBase.enumToConfKey(scope, configItem.self()), val.toString());
    }
  }
}
 

public static int run(String collection, Path input, Path output,
		Float regularizationFactor, Boolean addIntercept,
		Configuration baseConf) throws IOException, ClassNotFoundException,
		InterruptedException {
	Configuration conf = new Configuration(baseConf);
	if (null != addIntercept) {
		conf.setBoolean("lr.iteration.add.intercept", addIntercept);
	}
	if (null != regularizationFactor) {
		conf.setDouble("lr.iteration.regulariztion.factor",
				regularizationFactor);
	}

	conf.set("com.b5m.laser.msgpack.output.method", "update_online_model");

	Job job = Job.getInstance(conf);
	job.setJarByClass(LrIterationDriver.class);
	job.setJobName("logistic regression");

	FileInputFormat.setInputPaths(job, input);

	job.setOutputFormatClass(MsgpackOutputFormat.class);
	job.setOutputKeyClass(String.class);
	job.setOutputValueClass(LaserOnlineModel.class);

	LrIterationInputFormat.setNumMapTasks(job, 120);
	job.setInputFormatClass(LrIterationInputFormat.class);
	job.setMapperClass(LrIterationMapper.class);
	job.setNumReduceTasks(0);

	boolean succeeded = job.waitForCompletion(true);
	if (!succeeded) {
		throw new IllegalStateException("Job:logistic regression,  Failed!");
	}
	return 0;
}
 

@Test(timeout = 5000)
public void testAdditionalReserveFractionWeightedScaling() {
  Configuration conf = new Configuration(this.conf);
  conf.setStrings(TezJobConfig.TEZ_RUNTIME_SCALE_TASK_MEMORY_WEIGHTED_RATIOS,
      generateWeightStrings(2, 3, 6, 1, 1));
  conf.setDouble(TezJobConfig.TEZ_RUNTIME_SCALE_TASK_MEMORY_ADDITIONAL_RESERVATION_FRACTION_PER_IO, 0.025d);
  conf.setDouble(TezJobConfig.TEZ_RUNTIME_SCALE_TASK_MEMORY_ADDITIONAL_RESERVATION_FRACTION_MAX, 0.2d);

  MemoryDistributor dist = new MemoryDistributor(2, 2, conf);

  dist.setJvmMemory(10000l);

  // First request - ScatterGatherShuffleInput [weight 6]
  MemoryUpdateCallbackForTest e1Callback = new MemoryUpdateCallbackForTest();
  TezInputContext e1InputContext1 = createTestInputContext();
  InputDescriptor e1InDesc1 = createTestInputDescriptor(ShuffledMergedInput.class);
  dist.requestMemory(10000, e1Callback, e1InputContext1, e1InDesc1);

  // Second request - BroadcastInput [weight 2]
  MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest();
  TezInputContext e2InputContext2 = createTestInputContext();
  InputDescriptor e2InDesc2 = createTestInputDescriptor(ShuffledUnorderedKVInput.class);
  dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2);

  // Third request - randomOutput (simulates MROutput) [weight 1]
  MemoryUpdateCallbackForTest e3Callback = new MemoryUpdateCallbackForTest();
  TezOutputContext e3OutputContext1 = createTestOutputContext();
  OutputDescriptor e3OutDesc1 = createTestOutputDescriptor();
  dist.requestMemory(10000, e3Callback, e3OutputContext1, e3OutDesc1);

  // Fourth request - OnFileSortedOutput [weight 3]
  MemoryUpdateCallbackForTest e4Callback = new MemoryUpdateCallbackForTest();
  TezOutputContext e4OutputContext2 = createTestOutputContext();
  OutputDescriptor e4OutDesc2 = createTestOutputDescriptor(OnFileSortedOutput.class);
  dist.requestMemory(10000, e4Callback, e4OutputContext2, e4OutDesc2);

  dist.makeInitialAllocations();

  // Total available: 60% of 10K = 7000
  // 4 requests (weight) - 10K (6), 10K(2), 10K(1), 10K(3)
  // Scale down to - 3000, 1000, 500, 1500
  assertEquals(3000, e1Callback.assigned);
  assertEquals(1000, e2Callback.assigned);
  assertEquals(500, e3Callback.assigned);
  assertEquals(1500, e4Callback.assigned);
}
 

/**
 * In unpartitioned cartesian product, failure fraction should be #unique failure/#consumer that
 * depends on the src task. Here we test a 2x2 cartesian product and let 4th destination task fail.
 * The failure fraction limit is configured to be 0.25. So the failure fraction should be 1/2,
 * not 1/4.
 * @throws Exception
 */
@Test
public void testCartesianProduct() throws Exception {
  Configuration dagConf = new Configuration();
  dagConf.setDouble(TezConfiguration.TEZ_TASK_MAX_ALLOWED_OUTPUT_FAILURES_FRACTION, 0.25);
  DAG dag = DAG.create("dag");

  Configuration vertexConf = new Configuration();
  vertexConf.setInt(TestProcessor.getVertexConfName(
    TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v3"), 3);
  vertexConf.setInt(TestProcessor.getVertexConfName(
    TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v3"), 5);
  UserPayload vertexPayload = TezUtils.createUserPayloadFromConf(vertexConf);
  ProcessorDescriptor processorDescriptor =
    ProcessorDescriptor.create(TestProcessor.class.getName()).setUserPayload(vertexPayload);
  Vertex v1 = Vertex.create("v1", processorDescriptor, 2);
  Vertex v2 = Vertex.create("v2", processorDescriptor, 2);
  Vertex v3 = Vertex.create("v3", processorDescriptor);

  String[] sourceVertices = {"v1", "v2"};
  CartesianProductConfig cartesianProductConfig =
    new CartesianProductConfig(Arrays.asList(sourceVertices));
  TezConfiguration tezConf = new TezConfiguration();
  tezConf.setInt(CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_NUM_PARTITIONS, 1);
  tezConf.setBoolean(CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_ENABLE_GROUPING, false);
  UserPayload cartesianProductPayload =
    cartesianProductConfig.toUserPayload(tezConf);

  v3.setVertexManagerPlugin(
    VertexManagerPluginDescriptor.create(CartesianProductVertexManager.class.getName())
      .setUserPayload(cartesianProductPayload));

  EdgeManagerPluginDescriptor edgeManagerPluginDescriptor =
    EdgeManagerPluginDescriptor.create(CartesianProductEdgeManager.class.getName())
      .setUserPayload(cartesianProductPayload);

  Configuration inputConf = new Configuration();
  inputConf.setBoolean(TestInput.getVertexConfName(
    TestInput.TEZ_FAILING_INPUT_DO_FAIL, "v3"), true);
  inputConf.setInt(TestInput.getVertexConfName(
    TestInput.TEZ_FAILING_INPUT_FAILING_TASK_INDEX, "v3"), 3);
  inputConf.setInt(TestInput.getVertexConfName(
    TestInput.TEZ_FAILING_INPUT_FAILING_TASK_ATTEMPT, "v3"), 0);
  inputConf.setInt(TestInput.getVertexConfName(
    TestInput.TEZ_FAILING_INPUT_FAILING_INPUT_INDEX, "v3"), 0);
  inputConf.setInt(TestInput.getVertexConfName(
    TestInput.TEZ_FAILING_INPUT_FAILING_UPTO_INPUT_ATTEMPT, "v3"), 0);
  UserPayload inputPayload = TezUtils.createUserPayloadFromConf(inputConf);
  EdgeProperty edgeProperty =
    EdgeProperty.create(edgeManagerPluginDescriptor, DataMovementType.CUSTOM,
      DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, TestOutput.getOutputDesc(null),
      TestInput.getInputDesc(inputPayload));
  Edge e1 = Edge.create(v1, v3, edgeProperty);
  Edge e2 = Edge.create(v2, v3, edgeProperty);
  dag.addVertex(v1).addVertex(v2).addVertex(v3);
  dag.addEdge(e1).addEdge(e2);

  // run dag
  runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);
}
 

@Test(timeout = 5000)
public void testAdditionalReserveFractionWeightedScaling() throws TezException {
  Configuration conf = new Configuration(this.conf);
  conf.setStrings(TezConfiguration.TEZ_TASK_SCALE_MEMORY_WEIGHTED_RATIOS,
      WeightedScalingMemoryDistributor.generateWeightStrings(0, 0, 2, 3, 6, 1, 1));
  conf.setDouble(TezConfiguration.TEZ_TASK_SCALE_MEMORY_ADDITIONAL_RESERVATION_FRACTION_PER_IO, 0.025d);
  conf.setDouble(TezConfiguration.TEZ_TASK_SCALE_MEMORY_ADDITIONAL_RESERVATION_FRACTION_MAX, 0.2d);

  MemoryDistributor dist = new MemoryDistributor(2, 2, conf);

  dist.setJvmMemory(10000l);

  // First request - ScatterGatherShuffleInput [weight 6]
  MemoryUpdateCallbackForTest e1Callback = new MemoryUpdateCallbackForTest();
  InputContext e1InputContext1 = createTestInputContext();
  InputDescriptor e1InDesc1 = createTestInputDescriptor(OrderedGroupedKVInput.class);
  dist.requestMemory(10000, e1Callback, e1InputContext1, e1InDesc1);

  // Second request - BroadcastInput [weight 2]
  MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest();
  InputContext e2InputContext2 = createTestInputContext();
  InputDescriptor e2InDesc2 = createTestInputDescriptor(UnorderedKVInput.class);
  dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2);

  // Third request - randomOutput (simulates MROutput) [weight 1]
  MemoryUpdateCallbackForTest e3Callback = new MemoryUpdateCallbackForTest();
  OutputContext e3OutputContext1 = createTestOutputContext();
  OutputDescriptor e3OutDesc1 = createTestOutputDescriptor();
  dist.requestMemory(10000, e3Callback, e3OutputContext1, e3OutDesc1);

  // Fourth request - OnFileSortedOutput [weight 3]
  MemoryUpdateCallbackForTest e4Callback = new MemoryUpdateCallbackForTest();
  OutputContext e4OutputContext2 = createTestOutputContext();
  OutputDescriptor e4OutDesc2 = createTestOutputDescriptor(OrderedPartitionedKVOutput.class);
  dist.requestMemory(10000, e4Callback, e4OutputContext2, e4OutDesc2);

  dist.makeInitialAllocations();

  // Total available: 60% of 10K = 7000
  // 4 requests (weight) - 10K (6), 10K(2), 10K(1), 10K(3)
  // Scale down to - 3000, 1000, 500, 1500
  assertEquals(3000, e1Callback.assigned);
  assertEquals(1000, e2Callback.assigned);
  assertEquals(500, e3Callback.assigned);
  assertEquals(1500, e4Callback.assigned);
}
 

@Test(timeout = 5000)
public void testWeightedScalingNonConcurrent() throws TezException {
  Configuration conf = new Configuration(this.conf);
  conf.setBoolean(TezConfiguration.TEZ_TASK_SCALE_MEMORY_INPUT_OUTPUT_CONCURRENT, false);
  conf.setBoolean(TezConfiguration.TEZ_TASK_SCALE_MEMORY_NON_CONCURRENT_INPUTS_ENABLED, true);
  conf.setDouble(TezConfiguration.TEZ_TASK_SCALE_MEMORY_RESERVE_FRACTION, 0.2);
  conf.setStrings(TezConfiguration.TEZ_TASK_SCALE_MEMORY_WEIGHTED_RATIOS,
      WeightedScalingMemoryDistributor.generateWeightStrings(0, 0, 1, 2, 3, 1, 1));
  System.err.println(Joiner.on(",").join(conf.getStringCollection(
      TezConfiguration.TEZ_TASK_SCALE_MEMORY_WEIGHTED_RATIOS)));

  MemoryDistributor dist = new MemoryDistributor(2, 2, conf);

  dist.setJvmMemory(10000l);

  // First request - ScatterGatherShuffleInput
  MemoryUpdateCallbackForTest e1Callback = new MemoryUpdateCallbackForTest();
  InputContext e1InputContext1 = createTestInputContext();
  InputDescriptor e1InDesc1 = createTestInputDescriptor(OrderedGroupedKVInput.class);
  dist.requestMemory(10000, e1Callback, e1InputContext1, e1InDesc1);

  // Second request - BroadcastInput
  MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest();
  InputContext e2InputContext2 = createTestInputContext();
  InputDescriptor e2InDesc2 = createTestInputDescriptor(UnorderedKVInput.class);
  dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2);

  // Third request - randomOutput (simulates MROutput)
  MemoryUpdateCallbackForTest e3Callback = new MemoryUpdateCallbackForTest();
  OutputContext e3OutputContext1 = createTestOutputContext();
  OutputDescriptor e3OutDesc1 = createTestOutputDescriptor();
  dist.requestMemory(10000, e3Callback, e3OutputContext1, e3OutDesc1);

  // Fourth request - OnFileSortedOutput
  MemoryUpdateCallbackForTest e4Callback = new MemoryUpdateCallbackForTest();
  OutputContext e4OutputContext2 = createTestOutputContext();
  OutputDescriptor e4OutDesc2 = createTestOutputDescriptor(OrderedPartitionedKVOutput.class);
  dist.requestMemory(10000, e4Callback, e4OutputContext2, e4OutDesc2);

  // Fifth request - Processor
  MemoryUpdateCallbackForTest e5Callback = new MemoryUpdateCallbackForTest();
  ProcessorContext e5ProcContext = createTestProcessortContext();
  ProcessorDescriptor e5ProcDesc = createTestProcessorDescriptor();
  dist.requestMemory(10000, e5Callback, e5ProcContext, e5ProcDesc);

  dist.makeInitialAllocations();

  // Total available: 80% of 10K = 8000
  // 5 requests (weight) - 10K (3), 10K(1), 10K(1), 10K(2), 10K(1)
  // Overlap input and output memory
  assertEquals(5250, e1Callback.assigned);
  assertEquals(1750, e2Callback.assigned);
  assertEquals(2333, e3Callback.assigned);
  assertEquals(4666, e4Callback.assigned);
  assertEquals(1000, e5Callback.assigned);
}
 

@Test(timeout = 5000)
public void testAdditionalReserveFractionWeightedScalingNonConcurrent() throws TezException {
  Configuration conf = new Configuration(this.conf);
  conf.setBoolean(TezConfiguration.TEZ_TASK_SCALE_MEMORY_INPUT_OUTPUT_CONCURRENT, false);
  conf.setBoolean(TezConfiguration.TEZ_TASK_SCALE_MEMORY_NON_CONCURRENT_INPUTS_ENABLED, true);
  conf.setStrings(TezConfiguration.TEZ_TASK_SCALE_MEMORY_WEIGHTED_RATIOS,
      WeightedScalingMemoryDistributor.generateWeightStrings(0, 0, 2, 3, 6, 1, 1));
  conf.setDouble(TezConfiguration.TEZ_TASK_SCALE_MEMORY_ADDITIONAL_RESERVATION_FRACTION_PER_IO, 0.025d);
  conf.setDouble(TezConfiguration.TEZ_TASK_SCALE_MEMORY_ADDITIONAL_RESERVATION_FRACTION_MAX, 0.2d);

  MemoryDistributor dist = new MemoryDistributor(2, 2, conf);

  dist.setJvmMemory(10000l);

  // First request - ScatterGatherShuffleInput [weight 6]
  MemoryUpdateCallbackForTest e1Callback = new MemoryUpdateCallbackForTest();
  InputContext e1InputContext1 = createTestInputContext();
  InputDescriptor e1InDesc1 = createTestInputDescriptor(OrderedGroupedKVInput.class);
  dist.requestMemory(10000, e1Callback, e1InputContext1, e1InDesc1);

  // Second request - BroadcastInput [weight 2]
  MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest();
  InputContext e2InputContext2 = createTestInputContext();
  InputDescriptor e2InDesc2 = createTestInputDescriptor(UnorderedKVInput.class);
  dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2);

  // Third request - randomOutput (simulates MROutput) [weight 1]
  MemoryUpdateCallbackForTest e3Callback = new MemoryUpdateCallbackForTest();
  OutputContext e3OutputContext1 = createTestOutputContext();
  OutputDescriptor e3OutDesc1 = createTestOutputDescriptor();
  dist.requestMemory(10000, e3Callback, e3OutputContext1, e3OutDesc1);

  // Fourth request - OnFileSortedOutput [weight 3]
  MemoryUpdateCallbackForTest e4Callback = new MemoryUpdateCallbackForTest();
  OutputContext e4OutputContext2 = createTestOutputContext();
  OutputDescriptor e4OutDesc2 = createTestOutputDescriptor(OrderedPartitionedKVOutput.class);
  dist.requestMemory(10000, e4Callback, e4OutputContext2, e4OutDesc2);

  dist.makeInitialAllocations();

  // Total available: 60% of 10K = 6000
  // 4 requests (weight) - 10K (6), 10K(2), 10K(1), 10K(3)
  // Overlap input and output memory
  assertEquals(4500, e1Callback.assigned);
  assertEquals(1500, e2Callback.assigned);
  assertEquals(1500, e3Callback.assigned);
  assertEquals(4500, e4Callback.assigned);
}
 

@Test(timeout = 5000)
public void testWeightedScalingNonConcurrentInputsDisabled() throws TezException {
  Configuration conf = new Configuration(this.conf);
  conf.setBoolean(TezConfiguration.TEZ_TASK_SCALE_MEMORY_INPUT_OUTPUT_CONCURRENT, false);
  conf.setBoolean(TezConfiguration.TEZ_TASK_SCALE_MEMORY_NON_CONCURRENT_INPUTS_ENABLED, false);
  conf.setDouble(TezConfiguration.TEZ_TASK_SCALE_MEMORY_RESERVE_FRACTION, 0.2);
  conf.setStrings(TezConfiguration.TEZ_TASK_SCALE_MEMORY_WEIGHTED_RATIOS,
      WeightedScalingMemoryDistributor.generateWeightStrings(0, 0, 1, 2, 3, 1, 1));
  System.err.println(Joiner.on(",").join(conf.getStringCollection(
      TezConfiguration.TEZ_TASK_SCALE_MEMORY_WEIGHTED_RATIOS)));

  MemoryDistributor dist = new MemoryDistributor(2, 2, conf);

  dist.setJvmMemory(10000l);

  // First request - ScatterGatherShuffleInput
  MemoryUpdateCallbackForTest e1Callback = new MemoryUpdateCallbackForTest();
  InputContext e1InputContext1 = createTestInputContext();
  InputDescriptor e1InDesc1 = createTestInputDescriptor(OrderedGroupedKVInput.class);
  dist.requestMemory(10000, e1Callback, e1InputContext1, e1InDesc1);

  // Second request - BroadcastInput
  MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest();
  InputContext e2InputContext2 = createTestInputContext();
  InputDescriptor e2InDesc2 = createTestInputDescriptor(UnorderedKVInput.class);
  dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2);

  // Third request - randomOutput (simulates MROutput)
  MemoryUpdateCallbackForTest e3Callback = new MemoryUpdateCallbackForTest();
  OutputContext e3OutputContext1 = createTestOutputContext();
  OutputDescriptor e3OutDesc1 = createTestOutputDescriptor();
  dist.requestMemory(10000, e3Callback, e3OutputContext1, e3OutDesc1);

  // Fourth request - OnFileSortedOutput
  MemoryUpdateCallbackForTest e4Callback = new MemoryUpdateCallbackForTest();
  OutputContext e4OutputContext2 = createTestOutputContext();
  OutputDescriptor e4OutDesc2 = createTestOutputDescriptor(OrderedPartitionedKVOutput.class);
  dist.requestMemory(10000, e4Callback, e4OutputContext2, e4OutDesc2);

  // Fifth request - Processor
  MemoryUpdateCallbackForTest e5Callback = new MemoryUpdateCallbackForTest();
  ProcessorContext e5ProcContext = createTestProcessortContext();
  ProcessorDescriptor e5ProcDesc = createTestProcessorDescriptor();
  dist.requestMemory(10000, e5Callback, e5ProcContext, e5ProcDesc);

  dist.makeInitialAllocations();

  // Total available: 80% of 10K = 8000
  // 5 requests (weight) - 10K (3), 10K(1), 10K(1), 10K(2), 10K(1)
  // Overlap input and output memory
  assertEquals(3000, e1Callback.assigned);
  assertEquals(1000, e2Callback.assigned);
  assertEquals(2333, e3Callback.assigned);
  assertEquals(4666, e4Callback.assigned);
  assertEquals(1000, e5Callback.assigned);
}
 

/**
 * If there are two running InMemoryFlushRunnable, the later InMemoryFlushRunnable
 * may change the versionedList. And the first InMemoryFlushRunnable will use the chagned
 * versionedList to remove the corresponding segments.
 * In short, there will be some segements which isn't in merge are removed.
 * @throws IOException
 * @throws InterruptedException
 */
@Test
public void testRunDoubleMemStoreCompactors() throws IOException, InterruptedException {
  int flushSize = 500;
  Configuration conf = HBaseConfiguration.create();
  conf.set(HStore.MEMSTORE_CLASS_NAME, MyCompactingMemStoreWithCustomCompactor.class.getName());
  conf.setDouble(CompactingMemStore.IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, 0.25);
  MyCompactingMemStoreWithCustomCompactor.RUNNER_COUNT.set(0);
  conf.set(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, String.valueOf(flushSize));
  // Set the lower threshold to invoke the "MERGE" policy
  conf.set(MemStoreCompactionStrategy.COMPACTING_MEMSTORE_THRESHOLD_KEY, String.valueOf(0));
  init(name.getMethodName(), conf, ColumnFamilyDescriptorBuilder.newBuilder(family)
      .setInMemoryCompaction(MemoryCompactionPolicy.BASIC).build());
  byte[] value = Bytes.toBytes("thisisavarylargevalue");
  MemStoreSizing memStoreSizing = new NonThreadSafeMemStoreSizing();
  long ts = EnvironmentEdgeManager.currentTime();
  long seqId = 100;
  // older data whihc shouldn't be "seen" by client
  store.add(createCell(qf1, ts, seqId, value), memStoreSizing);
  store.add(createCell(qf2, ts, seqId, value), memStoreSizing);
  store.add(createCell(qf3, ts, seqId, value), memStoreSizing);
  assertEquals(1, MyCompactingMemStoreWithCustomCompactor.RUNNER_COUNT.get());
  StoreFlushContext storeFlushCtx = store.createFlushContext(id++, FlushLifeCycleTracker.DUMMY);
  storeFlushCtx.prepare();
  // This shouldn't invoke another in-memory flush because the first compactor thread
  // hasn't accomplished the in-memory compaction.
  store.add(createCell(qf1, ts + 1, seqId + 1, value), memStoreSizing);
  store.add(createCell(qf1, ts + 1, seqId + 1, value), memStoreSizing);
  store.add(createCell(qf1, ts + 1, seqId + 1, value), memStoreSizing);
  assertEquals(1, MyCompactingMemStoreWithCustomCompactor.RUNNER_COUNT.get());
  //okay. Let the compaction be completed
  MyMemStoreCompactor.START_COMPACTOR_LATCH.countDown();
  CompactingMemStore mem = (CompactingMemStore) ((HStore)store).memstore;
  while (mem.isMemStoreFlushingInMemory()) {
    TimeUnit.SECONDS.sleep(1);
  }
  // This should invoke another in-memory flush.
  store.add(createCell(qf1, ts + 2, seqId + 2, value), memStoreSizing);
  store.add(createCell(qf1, ts + 2, seqId + 2, value), memStoreSizing);
  store.add(createCell(qf1, ts + 2, seqId + 2, value), memStoreSizing);
  assertEquals(2, MyCompactingMemStoreWithCustomCompactor.RUNNER_COUNT.get());
  conf.set(HConstants.HREGION_MEMSTORE_FLUSH_SIZE,
    String.valueOf(TableDescriptorBuilder.DEFAULT_MEMSTORE_FLUSH_SIZE));
  storeFlushCtx.flushCache(Mockito.mock(MonitoredTask.class));
  storeFlushCtx.commit(Mockito.mock(MonitoredTask.class));
}
 

@Test
public void testInsufficientlyReportedTableFiltering() throws Exception {
  final Map<TableName,Integer> reportedRegions = new HashMap<>();
  final Map<TableName,Integer> actualRegions = new HashMap<>();
  final Configuration conf = HBaseConfiguration.create();
  conf.setDouble(QuotaObserverChore.QUOTA_OBSERVER_CHORE_REPORT_PERCENT_KEY, 0.95);

  TableName tooFewRegionsTable = TableName.valueOf("tn1");
  TableName sufficientRegionsTable = TableName.valueOf("tn2");
  TableName tooFewRegionsNamespaceTable = TableName.valueOf("ns1", "tn2");
  TableName sufficientRegionsNamespaceTable = TableName.valueOf("ns1", "tn2");
  final TablesWithQuotas tables = new TablesWithQuotas(conn, conf) {
    @Override
    Configuration getConfiguration() {
      return conf;
    }

    @Override
    int getNumRegions(TableName tableName) {
      return actualRegions.get(tableName);
    }

    @Override
    int getNumReportedRegions(TableName table, QuotaSnapshotStore<TableName> tableStore) {
      return reportedRegions.get(table);
    }
  };
  tables.addTableQuotaTable(tooFewRegionsTable);
  tables.addTableQuotaTable(sufficientRegionsTable);
  tables.addNamespaceQuotaTable(tooFewRegionsNamespaceTable);
  tables.addNamespaceQuotaTable(sufficientRegionsNamespaceTable);

  reportedRegions.put(tooFewRegionsTable, 5);
  actualRegions.put(tooFewRegionsTable, 10);
  reportedRegions.put(sufficientRegionsTable, 19);
  actualRegions.put(sufficientRegionsTable, 20);
  reportedRegions.put(tooFewRegionsNamespaceTable, 9);
  actualRegions.put(tooFewRegionsNamespaceTable, 10);
  reportedRegions.put(sufficientRegionsNamespaceTable, 98);
  actualRegions.put(sufficientRegionsNamespaceTable, 100);

  // Unused argument
  tables.filterInsufficientlyReportedTables(null);
  Set<TableName> filteredTablesWithTableQuotas = tables.getTableQuotaTables();
  assertEquals(Collections.singleton(sufficientRegionsTable), filteredTablesWithTableQuotas);
  Set<TableName> filteredTablesWithNamespaceQutoas = tables.getNamespaceQuotaTables();
  assertEquals(Collections.singleton(sufficientRegionsNamespaceTable), filteredTablesWithNamespaceQutoas);
}