Java Code Examples for org.apache.helix.model.ExternalView#setState()

The following examples show how to use org.apache.helix.model.ExternalView#setState() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: TestInstanceValidationUtil.java From helix with Apache License 2.0

6 votes

private ExternalView prepareExternalView() {
  ExternalView externalView = new ExternalView(RESOURCE_NAME);
  externalView.getRecord()
      .setSimpleField(ExternalView.ExternalViewProperty.STATE_MODEL_DEF_REF.toString(),
          MasterSlaveSMD.name);
  externalView.setState("p1", "h1", "MASTER");
  externalView.setState("p1", "h2", "SLAVE");
  externalView.setState("p1", "h3", "SLAVE");

  externalView.setState("p2", "h1", "SLAVE");
  externalView.setState("p2", "h2", "MASTER");
  externalView.setState("p2", "h3", "SLAVE");

  externalView.setState("p3", "h1", "SLAVE");
  externalView.setState("p3", "h2", "MASTER");
  externalView.setState("p3", "h3", "SLAVE");

  return externalView;
}

Example 2

Source File: HelixExternalViewBasedQueryQuotaManagerTest.java From incubator-pinot with Apache License 2.0

5 votes

@Test
public void testBothTableHaveQpsQuotaConfig()
    throws Exception {
  ExternalView brokerResource = generateBrokerResource(OFFLINE_TABLE_NAME);
  brokerResource.setState(REALTIME_TABLE_NAME, BROKER_INSTANCE_ID, "ONLINE");
  brokerResource.setState(REALTIME_TABLE_NAME, "broker_instance_2", "OFFLINE");

  QuotaConfig quotaConfig = new QuotaConfig("6G", "100.00");
  TableConfig realtimeTableConfig =
      new TableConfigBuilder(TableType.REALTIME).setTableName(RAW_TABLE_NAME).setQuotaConfig(quotaConfig)
          .setRetentionTimeUnit("DAYS").setRetentionTimeValue("1").setSegmentPushType("APPEND")
          .setBrokerTenant("testBroker").setServerTenant("testServer").build();
  TableConfig offlineTableConfig =
      new TableConfigBuilder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME).setQuotaConfig(quotaConfig)
          .setRetentionTimeUnit("DAYS").setRetentionTimeValue("1").setSegmentPushType("APPEND")
          .setBrokerTenant("testBroker").setServerTenant("testServer").build();

  ZKMetadataProvider.setRealtimeTableConfig(_testPropertyStore, REALTIME_TABLE_NAME,
      TableConfigUtils.toZNRecord(realtimeTableConfig));
  ZKMetadataProvider
      .setOfflineTableConfig(_testPropertyStore, OFFLINE_TABLE_NAME, TableConfigUtils.toZNRecord(offlineTableConfig));

  // Since each table has 2 online brokers, per broker rate becomes 100.0 / 2 = 50.0
  _queryQuotaManager.initTableQueryQuota(offlineTableConfig, brokerResource);
  Assert.assertEquals(_queryQuotaManager.getRateLimiterMapSize(), 1);
  _queryQuotaManager.initTableQueryQuota(realtimeTableConfig, brokerResource);
  // The hash map now contains 2 entries for both of the tables.
  Assert.assertEquals(_queryQuotaManager.getRateLimiterMapSize(), 2);

  // Rate limiter generates 1 token every 10 milliseconds, have to make it sleep for a while.
  runQueries(70, 10L);

  _queryQuotaManager.dropTableQueryQuota(OFFLINE_TABLE_NAME);
  // Since real-time table still has the qps quota, the size of the hash map becomes 1.
  Assert.assertEquals(_queryQuotaManager.getRateLimiterMapSize(), 1);

  _queryQuotaManager.dropTableQueryQuota(REALTIME_TABLE_NAME);
  // Since the only 1 table which has qps quota has been dropped, the size of the hash map becomes 0.
  Assert.assertEquals(_queryQuotaManager.getRateLimiterMapSize(), 0);
}

Example 3

Source File: HelixExternalViewBasedQueryQuotaManagerTest.java From incubator-pinot with Apache License 2.0

5 votes

@Test
public void testNoOnlineBrokerServiceOnBrokerResource()
    throws Exception {
  ExternalView brokerResource = new ExternalView(CommonConstants.Helix.BROKER_RESOURCE_INSTANCE);
  brokerResource.setState(OFFLINE_TABLE_NAME, "broker_instance_2", "OFFLINE");
  TableConfig tableConfig = generateDefaultTableConfig(OFFLINE_TABLE_NAME);
  ZKMetadataProvider
      .setOfflineTableConfig(_testPropertyStore, OFFLINE_TABLE_NAME, TableConfigUtils.toZNRecord(tableConfig));
  setQps(tableConfig);
  _queryQuotaManager.initTableQueryQuota(tableConfig, brokerResource);

  // For the 1st version we don't check the number of online brokers.
  // Thus the expected size now is 1. It'll be 0 when we bring dynamic rate back.
  Assert.assertEquals(_queryQuotaManager.getRateLimiterMapSize(), 1);
}

Example 4

Source File: ExternalViewComputeStage.java From helix with Apache License 2.0

4 votes

private void computeExternalView(final Resource resource,
    final CurrentStateOutput currentStateOutput, final ResourceControllerDataProvider cache,
    final ClusterStatusMonitor clusterStatusMonitor, final Map<String, ExternalView> curExtViews,
    final HelixManager manager, Set<String> monitoringResources, List<ExternalView> newExtViews) {
  String resourceName = resource.getResourceName();
  ExternalView view = new ExternalView(resource.getResourceName());
  // if resource ideal state has bucket size, set it
  // otherwise resource has been dropped, use bucket size from current state instead
  if (resource.getBucketSize() > 0) {
    view.setBucketSize(resource.getBucketSize());
  } else {
    view.setBucketSize(currentStateOutput.getBucketSize(resourceName));
  }

  int totalPendingMessageCount = 0;

  for (Partition partition : resource.getPartitions()) {
    Map<String, String> currentStateMap =
        currentStateOutput.getCurrentStateMap(resourceName, partition);
    if (currentStateMap != null && currentStateMap.size() > 0) {
      for (String instance : currentStateMap.keySet()) {
        view.setState(partition.getPartitionName(), instance, currentStateMap.get(instance));
      }
    }
    totalPendingMessageCount +=
        currentStateOutput.getPendingMessageMap(resource.getResourceName(), partition).size();
  }

  // Update cluster status monitor mbean
  IdealState idealState = cache.getIdealState(resourceName);
  ResourceConfig resourceConfig = cache.getResourceConfig(resourceName);
  if (clusterStatusMonitor != null) {
    if (idealState != null // has ideal state
        && (resourceConfig == null || !resourceConfig.isMonitoringDisabled()) // monitoring not disabled
        && !idealState.getStateModelDefRef() // and not a job resource
        .equalsIgnoreCase(DefaultSchedulerMessageHandlerFactory.SCHEDULER_TASK_QUEUE)) {
      StateModelDefinition stateModelDef =
          cache.getStateModelDef(idealState.getStateModelDefRef());
      clusterStatusMonitor
          .setResourceStatus(view, cache.getIdealState(view.getResourceName()),
              stateModelDef, totalPendingMessageCount);
      monitoringResources.add(resourceName);
    }
  }

  ExternalView curExtView = curExtViews.get(resourceName);
  // copy simplefields from IS, in cases where IS is deleted copy it from existing ExternalView
  if (idealState != null) {
    view.getRecord().getSimpleFields().putAll(idealState.getRecord().getSimpleFields());
  } else if (curExtView != null) {
    view.getRecord().getSimpleFields().putAll(curExtView.getRecord().getSimpleFields());
  }

  // compare the new external view with current one, set only on different
  if (curExtView == null || !curExtView.getRecord().equals(view.getRecord())) {
    // Add external view to the list which will be written to ZK later.
    newExtViews.add(view);

    // For SCHEDULER_TASK_RESOURCE resource group (helix task queue), we need to find out which
    // task partitions are finished (COMPLETED or ERROR), update the status update of the original
    // scheduler message, and then remove the partitions from the ideal state
    if (idealState != null
        && idealState.getStateModelDefRef().equalsIgnoreCase(
        DefaultSchedulerMessageHandlerFactory.SCHEDULER_TASK_QUEUE)) {
      updateScheduledTaskStatus(view, manager, idealState);
    }
  }
}

Example 5

Source File: SegmentStatusCheckerTest.java From incubator-pinot with Apache License 2.0

4 votes

@Test
public void offlineBasicTest()
    throws Exception {
  final String tableName = "myTable_OFFLINE";
  List<String> allTableNames = new ArrayList<String>();
  allTableNames.add(tableName);
  IdealState idealState = new IdealState(tableName);
  idealState.setPartitionState("myTable_0", "pinot1", "ONLINE");
  idealState.setPartitionState("myTable_0", "pinot2", "ONLINE");
  idealState.setPartitionState("myTable_0", "pinot3", "ONLINE");
  idealState.setPartitionState("myTable_1", "pinot1", "ONLINE");
  idealState.setPartitionState("myTable_1", "pinot2", "ONLINE");
  idealState.setPartitionState("myTable_1", "pinot3", "ONLINE");
  idealState.setPartitionState("myTable_2", "pinot3", "OFFLINE");
  idealState.setReplicas("2");
  idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED);

  ExternalView externalView = new ExternalView(tableName);
  externalView.setState("myTable_0", "pinot1", "ONLINE");
  externalView.setState("myTable_0", "pinot2", "ONLINE");
  externalView.setState("myTable_1", "pinot1", "ERROR");
  externalView.setState("myTable_1", "pinot2", "ONLINE");

  {
    helixResourceManager = mock(PinotHelixResourceManager.class);
    when(helixResourceManager.getAllTables()).thenReturn(allTableNames);
    when(helixResourceManager.getTableIdealState(tableName)).thenReturn(idealState);
    when(helixResourceManager.getTableExternalView(tableName)).thenReturn(externalView);
  }
  {
    config = mock(ControllerConf.class);
    when(config.getStatusCheckerFrequencyInSeconds()).thenReturn(300);
    when(config.getStatusCheckerWaitForPushTimeInSeconds()).thenReturn(300);
  }
  {
    leadControllerManager = mock(LeadControllerManager.class);
    when(leadControllerManager.isLeaderForTable(anyString())).thenReturn(true);
  }
  metricsRegistry = new MetricsRegistry();
  controllerMetrics = new ControllerMetrics(metricsRegistry);
  segmentStatusChecker =
      new SegmentStatusChecker(helixResourceManager, leadControllerManager, config, controllerMetrics);
  segmentStatusChecker.start();
  segmentStatusChecker.run();
  Assert.assertEquals(
      controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.SEGMENTS_IN_ERROR_STATE), 1);
  Assert
      .assertEquals(controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.NUMBER_OF_REPLICAS),
          1);
  Assert
      .assertEquals(controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.PERCENT_OF_REPLICAS),
          33);
  Assert.assertEquals(
      controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.PERCENT_SEGMENTS_AVAILABLE), 100);
}

Example 6

Source File: SegmentStatusCheckerTest.java From incubator-pinot with Apache License 2.0

4 votes

@Test
public void realtimeBasicTest()
    throws Exception {
  final String tableName = "myTable_REALTIME";
  final String rawTableName = TableNameBuilder.extractRawTableName(tableName);
  List<String> allTableNames = new ArrayList<String>();
  allTableNames.add(tableName);
  final LLCSegmentName seg1 = new LLCSegmentName(rawTableName, 1, 0, System.currentTimeMillis());
  final LLCSegmentName seg2 = new LLCSegmentName(rawTableName, 1, 1, System.currentTimeMillis());
  final LLCSegmentName seg3 = new LLCSegmentName(rawTableName, 2, 1, System.currentTimeMillis());
  IdealState idealState = new IdealState(tableName);
  idealState.setPartitionState(seg1.getSegmentName(), "pinot1", "ONLINE");
  idealState.setPartitionState(seg1.getSegmentName(), "pinot2", "ONLINE");
  idealState.setPartitionState(seg1.getSegmentName(), "pinot3", "ONLINE");
  idealState.setPartitionState(seg2.getSegmentName(), "pinot1", "ONLINE");
  idealState.setPartitionState(seg2.getSegmentName(), "pinot2", "ONLINE");
  idealState.setPartitionState(seg2.getSegmentName(), "pinot3", "ONLINE");
  idealState.setPartitionState(seg3.getSegmentName(), "pinot1", "CONSUMING");
  idealState.setPartitionState(seg3.getSegmentName(), "pinot2", "CONSUMING");
  idealState.setPartitionState(seg3.getSegmentName(), "pinot3", "OFFLINE");
  idealState.setReplicas("3");
  idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED);

  ExternalView externalView = new ExternalView(tableName);
  externalView.setState(seg1.getSegmentName(), "pinot1", "ONLINE");
  externalView.setState(seg1.getSegmentName(), "pinot2", "ONLINE");
  externalView.setState(seg1.getSegmentName(), "pinot3", "ONLINE");
  externalView.setState(seg2.getSegmentName(), "pinot1", "CONSUMING");
  externalView.setState(seg2.getSegmentName(), "pinot2", "ONLINE");
  externalView.setState(seg2.getSegmentName(), "pinot3", "CONSUMING");
  externalView.setState(seg3.getSegmentName(), "pinot1", "CONSUMING");
  externalView.setState(seg3.getSegmentName(), "pinot2", "CONSUMING");
  externalView.setState(seg3.getSegmentName(), "pinot3", "OFFLINE");

  {
    helixResourceManager = mock(PinotHelixResourceManager.class);
    when(helixResourceManager.getAllTables()).thenReturn(allTableNames);
    when(helixResourceManager.getTableIdealState(tableName)).thenReturn(idealState);
    when(helixResourceManager.getTableExternalView(tableName)).thenReturn(externalView);
  }
  {
    config = mock(ControllerConf.class);
    when(config.getStatusCheckerFrequencyInSeconds()).thenReturn(300);
    when(config.getStatusCheckerWaitForPushTimeInSeconds()).thenReturn(300);
  }
  {
    leadControllerManager = mock(LeadControllerManager.class);
    when(leadControllerManager.isLeaderForTable(anyString())).thenReturn(true);
  }
  metricsRegistry = new MetricsRegistry();
  controllerMetrics = new ControllerMetrics(metricsRegistry);
  segmentStatusChecker =
      new SegmentStatusChecker(helixResourceManager, leadControllerManager, config, controllerMetrics);
  segmentStatusChecker.start();
  segmentStatusChecker.run();
  Assert.assertEquals(
      controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.SEGMENTS_IN_ERROR_STATE), 0);
  Assert
      .assertEquals(controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.NUMBER_OF_REPLICAS),
          3);
  Assert
      .assertEquals(controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.PERCENT_OF_REPLICAS),
          100);
  Assert.assertEquals(
      controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.PERCENT_SEGMENTS_AVAILABLE), 100);
}

Example 7

Source File: SegmentStatusCheckerTest.java From incubator-pinot with Apache License 2.0

4 votes

@Test
public void missingEVPartitionTest()
    throws Exception {
  final String tableName = "myTable_OFFLINE";
  List<String> allTableNames = new ArrayList<String>();
  allTableNames.add(tableName);
  IdealState idealState = new IdealState(tableName);
  idealState.setPartitionState("myTable_0", "pinot1", "ONLINE");
  idealState.setPartitionState("myTable_0", "pinot2", "ONLINE");
  idealState.setPartitionState("myTable_0", "pinot3", "ONLINE");
  idealState.setPartitionState("myTable_1", "pinot1", "ONLINE");
  idealState.setPartitionState("myTable_1", "pinot2", "ONLINE");
  idealState.setPartitionState("myTable_1", "pinot3", "ONLINE");
  idealState.setPartitionState("myTable_2", "pinot3", "OFFLINE");
  idealState.setPartitionState("myTable_3", "pinot3", "ONLINE");
  idealState.setReplicas("2");
  idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED);

  ExternalView externalView = new ExternalView(tableName);
  externalView.setState("myTable_0", "pinot1", "ONLINE");
  externalView.setState("myTable_0", "pinot2", "ONLINE");
  externalView.setState("myTable_1", "pinot1", "ERROR");
  externalView.setState("myTable_1", "pinot2", "ONLINE");

  ZNRecord znrecord = new ZNRecord("myTable_0");
  znrecord.setSimpleField(CommonConstants.Segment.SEGMENT_NAME, "myTable_0");
  znrecord.setSimpleField(CommonConstants.Segment.TABLE_NAME, "myTable_OFFLINE");
  znrecord.setSimpleField(CommonConstants.Segment.INDEX_VERSION, "v1");
  znrecord.setEnumField(CommonConstants.Segment.SEGMENT_TYPE, CommonConstants.Segment.SegmentType.OFFLINE);
  znrecord.setLongField(CommonConstants.Segment.START_TIME, 1000);
  znrecord.setLongField(CommonConstants.Segment.END_TIME, 2000);
  znrecord.setSimpleField(CommonConstants.Segment.TIME_UNIT, TimeUnit.HOURS.toString());
  znrecord.setLongField(CommonConstants.Segment.TOTAL_DOCS, 10000);
  znrecord.setLongField(CommonConstants.Segment.CRC, 1234);
  znrecord.setLongField(CommonConstants.Segment.CREATION_TIME, 3000);
  znrecord.setSimpleField(CommonConstants.Segment.Offline.DOWNLOAD_URL, "http://localhost:8000/myTable_0");
  znrecord.setLongField(CommonConstants.Segment.Offline.PUSH_TIME, System.currentTimeMillis());
  znrecord.setLongField(CommonConstants.Segment.Offline.REFRESH_TIME, System.currentTimeMillis());

  ZkHelixPropertyStore<ZNRecord> propertyStore;
  {
    propertyStore = (ZkHelixPropertyStore<ZNRecord>) mock(ZkHelixPropertyStore.class);
    when(propertyStore.get("/SEGMENTS/myTable_OFFLINE/myTable_3", null, AccessOption.PERSISTENT))
        .thenReturn(znrecord);
  }

  {
    helixResourceManager = mock(PinotHelixResourceManager.class);
    when(helixResourceManager.getAllTables()).thenReturn(allTableNames);
    when(helixResourceManager.getTableIdealState(tableName)).thenReturn(idealState);
    when(helixResourceManager.getTableExternalView(tableName)).thenReturn(externalView);
    when(helixResourceManager.getOfflineSegmentZKMetadata(tableName, "myTable_3"))
        .thenReturn(new OfflineSegmentZKMetadata(znrecord));
  }
  {
    config = mock(ControllerConf.class);
    when(config.getStatusCheckerFrequencyInSeconds()).thenReturn(300);
    when(config.getStatusCheckerWaitForPushTimeInSeconds()).thenReturn(0);
  }
  {
    leadControllerManager = mock(LeadControllerManager.class);
    when(leadControllerManager.isLeaderForTable(anyString())).thenReturn(true);
  }
  metricsRegistry = new MetricsRegistry();
  controllerMetrics = new ControllerMetrics(metricsRegistry);
  segmentStatusChecker =
      new SegmentStatusChecker(helixResourceManager, leadControllerManager, config, controllerMetrics);
  segmentStatusChecker.start();
  segmentStatusChecker.run();
  Assert.assertEquals(
      controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.SEGMENTS_IN_ERROR_STATE), 1);
  Assert
      .assertEquals(controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.NUMBER_OF_REPLICAS),
          0);
  Assert.assertEquals(
      controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.PERCENT_SEGMENTS_AVAILABLE), 75);
}

Example 8

Source File: SegmentStatusCheckerTest.java From incubator-pinot with Apache License 2.0

4 votes

@Test
public void missingEVPartitionPushTest()
    throws Exception {
  final String tableName = "myTable_OFFLINE";
  List<String> allTableNames = new ArrayList<String>();
  allTableNames.add(tableName);
  IdealState idealState = new IdealState(tableName);
  idealState.setPartitionState("myTable_0", "pinot1", "ONLINE");
  idealState.setPartitionState("myTable_1", "pinot1", "ONLINE");
  idealState.setPartitionState("myTable_1", "pinot2", "ONLINE");
  idealState.setReplicas("2");
  idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED);

  ExternalView externalView = new ExternalView(tableName);
  externalView.setState("myTable_1", "pinot1", "ONLINE");
  externalView.setState("myTable_1", "pinot2", "ONLINE");

  ZNRecord znrecord = new ZNRecord("myTable_0");
  znrecord.setSimpleField(CommonConstants.Segment.SEGMENT_NAME, "myTable_0");
  znrecord.setSimpleField(CommonConstants.Segment.TABLE_NAME, "myTable_OFFLINE");
  znrecord.setSimpleField(CommonConstants.Segment.INDEX_VERSION, "v1");
  znrecord.setEnumField(CommonConstants.Segment.SEGMENT_TYPE, CommonConstants.Segment.SegmentType.OFFLINE);
  znrecord.setLongField(CommonConstants.Segment.START_TIME, 1000);
  znrecord.setLongField(CommonConstants.Segment.END_TIME, 2000);
  znrecord.setSimpleField(CommonConstants.Segment.TIME_UNIT, TimeUnit.HOURS.toString());
  znrecord.setLongField(CommonConstants.Segment.TOTAL_DOCS, 10000);
  znrecord.setLongField(CommonConstants.Segment.CRC, 1234);
  znrecord.setLongField(CommonConstants.Segment.CREATION_TIME, 3000);
  znrecord.setSimpleField(CommonConstants.Segment.Offline.DOWNLOAD_URL, "http://localhost:8000/myTable_0");
  znrecord.setLongField(CommonConstants.Segment.Offline.PUSH_TIME, System.currentTimeMillis());
  znrecord.setLongField(CommonConstants.Segment.Offline.REFRESH_TIME, System.currentTimeMillis());

  {
    helixResourceManager = mock(PinotHelixResourceManager.class);
    when(helixResourceManager.getAllTables()).thenReturn(allTableNames);
    when(helixResourceManager.getTableIdealState(tableName)).thenReturn(idealState);
    when(helixResourceManager.getTableExternalView(tableName)).thenReturn(externalView);
    when(helixResourceManager.getOfflineSegmentZKMetadata(tableName, "myTable_0"))
        .thenReturn(new OfflineSegmentZKMetadata(znrecord));
  }
  {
    config = mock(ControllerConf.class);
    when(config.getStatusCheckerFrequencyInSeconds()).thenReturn(300);
    when(config.getStatusCheckerWaitForPushTimeInSeconds()).thenReturn(300);
  }
  {
    leadControllerManager = mock(LeadControllerManager.class);
    when(leadControllerManager.isLeaderForTable(anyString())).thenReturn(true);
  }
  metricsRegistry = new MetricsRegistry();
  controllerMetrics = new ControllerMetrics(metricsRegistry);
  segmentStatusChecker =
      new SegmentStatusChecker(helixResourceManager, leadControllerManager, config, controllerMetrics);
  segmentStatusChecker.start();
  segmentStatusChecker.run();
  Assert.assertEquals(
      controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.SEGMENTS_IN_ERROR_STATE), 0);
  Assert
      .assertEquals(controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.NUMBER_OF_REPLICAS),
          2);
  Assert
      .assertEquals(controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.PERCENT_OF_REPLICAS),
          100);
  Assert.assertEquals(
      controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.PERCENT_SEGMENTS_AVAILABLE), 100);
}

Example 9

Source File: HelixExternalViewBasedQueryQuotaManagerTest.java From incubator-pinot with Apache License 2.0

4 votes

private ExternalView generateBrokerResource(String tableName) {
  ExternalView brokerResource = new ExternalView(CommonConstants.Helix.BROKER_RESOURCE_INSTANCE);
  brokerResource.setState(tableName, BROKER_INSTANCE_ID, "ONLINE");
  brokerResource.setState(tableName, "broker_instance_2", "OFFLINE");
  return brokerResource;
}

Example 10

Source File: ServiceStatusTest.java From incubator-pinot with Apache License 2.0

4 votes

private void testMultipleResourcesAndPercent(double percentReady) {
  final long now = System.currentTimeMillis();
  random = new Random(now);
  final String clusterName = "noSuchCluster";
  final List<String> tables = new ArrayList<>();
  final int tableCount = 2500 + random.nextInt(100);
  int readyTables = 0;
  Map<String, IdealState> idealStates = new HashMap<>();
  Map<String, ExternalView> externalViews = new HashMap<>();

  for (int i = 1; i <= tableCount; i++) {
    final String tableName = generateRandomString(10) + String.valueOf(i);
    tables.add(tableName);
    final String segmentName = "segment1";
    String evState;
    if (random.nextDouble() * 100 < percentReady) {
      evState = "ONLINE";
      readyTables++;
    } else {
      evState = "OFFLINE";
    }
    IdealState idealState = new IdealState(tableName);
    idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED);
    idealState.setPartitionState(segmentName, INSTANCE_NAME, "ONLINE");
    ExternalView externalView = new ExternalView(tableName);
    externalView.setState(segmentName, INSTANCE_NAME, evState);
    idealStates.put(tableName, idealState);
    externalViews.put(tableName, externalView);
  }

  final double actualReadyPercent = (double) readyTables * 100 / tableCount;
  double lowestReadyPercent = (int) Math.round(actualReadyPercent);
  lowestReadyPercent = lowestReadyPercent > 2 ? lowestReadyPercent - 2 : 1;  // Should be 2 below  percentReady

  // Create ServiceCallback objects with minReadyPercent set to values between lowestReadyPercent and 100.
  // Call getServiceStatus() enough number of times so that we are only left with the tables
  // that are not ready yet. We need to call getServiceStatus() at most tableCount times.
  for (double minReadyPercent = lowestReadyPercent; minReadyPercent <= 100; minReadyPercent += 0.1) {
    TestMultiResourceISAndEVMatchCB callback =
        new TestMultiResourceISAndEVMatchCB(clusterName, INSTANCE_NAME, tables, minReadyPercent);
    callback.setIdealStates(idealStates);
    callback.setExternalViews(externalViews);

    ServiceStatus.Status status = callback.getServiceStatus();
    // we need to call getServiceStatus() at most the number of bad tables plus 1,
    // to get a STARTED condition if we can. After that, the return value should
    // never change.
    final int nBadTables = tableCount - readyTables;
    for (int i = 0; i <= nBadTables; i++) {
      status = callback.getServiceStatus();
    }

    ServiceStatus.Status expected =
        minReadyPercent > actualReadyPercent ? ServiceStatus.Status.STARTING : ServiceStatus.Status.GOOD;
    String errorMsg = "Mismatch at " + minReadyPercent + "%, tableCount=" + tableCount + ", percentTablesReady="
        + actualReadyPercent + ":" + callback.getStatusDescription();
    Assert.assertEquals(status, expected, errorMsg);

    // The status should never change going forward from here.
    for (int i = nBadTables + 1; i < tableCount; i++) {
      ServiceStatus.Status laterStatus = callback.getServiceStatus();
      String msg = "Mismatch at " + minReadyPercent + "%, tableCount=" + tableCount + ", percentTablesReady="
          + actualReadyPercent + ", i=" + i + ":" + callback.getStatusDescription();
      Assert.assertEquals(laterStatus, status, msg);
    }
  }
}