Java Code Examples for org.apache.helix.model.IdealState#setPartitionState()
The following examples show how to use
org.apache.helix.model.IdealState#setPartitionState() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: ZkTestBase.java From helix with Apache License 2.0 | 6 votes |
|
protected IdealState createIdealState(String resourceGroupName, String instanceGroupTag,
List<String> instanceNames, int numPartition, int replica, String rebalanceMode,
String stateModelDef) {
IdealState is = _gSetupTool.createIdealStateForResourceGroup(resourceGroupName,
instanceGroupTag, numPartition, replica, rebalanceMode, stateModelDef);
// setup initial partition->instance mapping.
int nodeIdx = 0;
int numNode = instanceNames.size();
assert (numNode >= replica);
for (int i = 0; i < numPartition; i++) {
String partitionName = resourceGroupName + "_" + i;
for (int j = 0; j < replica; j++) {
is.setPartitionState(partitionName, instanceNames.get((nodeIdx + j) % numNode),
OnlineOfflineSMD.States.ONLINE.toString());
}
nodeIdx++;
}
return is;
}
Example 2
| Source File: TestCustomRebalancer.java From helix with Apache License 2.0 | 5 votes |
|
/**
* This test was written because there is an edge case where an instance becomes disabled while a
* partition is bootstrapping by way of pending
* messages.
* The newly bootstrapped partitions never get further state transitions because the instance
* won't ever get added to instanceStateMap (this issue has been fixed). In other words, if there
* are mapping changes while a partition is bootstrapping, the final state should go into the best
* possible mapping for clusters to converge correctly.
*/
@Test
public void testDisabledBootstrappingPartitions() {
String resourceName = "Test";
String partitionName = "Test0";
String instanceName = "localhost";
String stateModelName = "OnlineOffline";
StateModelDefinition stateModelDef = new OnlineOfflineSMD();
IdealState idealState = new IdealState(resourceName);
idealState.setStateModelDefRef(stateModelName);
idealState.setPartitionState(partitionName, instanceName, "ONLINE");
Resource resource = new Resource(resourceName);
resource.addPartition(partitionName);
CustomRebalancer customRebalancer = new CustomRebalancer();
ResourceControllerDataProvider cache = mock(ResourceControllerDataProvider.class);
when(cache.getStateModelDef(stateModelName)).thenReturn(stateModelDef);
when(cache.getDisabledInstancesForPartition(resource.getResourceName(), partitionName))
.thenReturn(ImmutableSet.of(instanceName));
when(cache.getLiveInstances())
.thenReturn(ImmutableMap.of(instanceName, new LiveInstance(instanceName)));
CurrentStateOutput currOutput = new CurrentStateOutput();
ResourceAssignment resourceAssignment =
customRebalancer.computeBestPossiblePartitionState(cache, idealState, resource, currOutput);
Assert.assertEquals(
resourceAssignment.getReplicaMap(new Partition(partitionName)).get(instanceName),
stateModelDef.getInitialState());
}
Example 3
| Source File: PinotHelixResourceManager.java From incubator-pinot with Apache License 2.0 | 5 votes |
|
private void addInstanceToBrokerIdealState(String brokerTenantTag, String instanceName) {
IdealState tableIdealState = _helixAdmin.getResourceIdealState(_helixClusterName, Helix.BROKER_RESOURCE_INSTANCE);
for (String tableNameWithType : tableIdealState.getPartitionSet()) {
TableConfig tableConfig = ZKMetadataProvider.getTableConfig(_propertyStore, tableNameWithType);
Preconditions.checkNotNull(tableConfig);
String brokerTag = TagNameUtils.extractBrokerTag(tableConfig.getTenantConfig());
if (brokerTag.equals(brokerTenantTag)) {
tableIdealState.setPartitionState(tableNameWithType, instanceName, BrokerResourceStateModel.ONLINE);
}
}
_helixAdmin.setResourceIdealState(_helixClusterName, Helix.BROKER_RESOURCE_INSTANCE, tableIdealState);
}
Example 4
| Source File: SegmentStatusCheckerTest.java From incubator-pinot with Apache License 2.0 | 5 votes |
|
@Test
public void noReplicas()
throws Exception {
final String tableName = "myTable_REALTIME";
List<String> allTableNames = new ArrayList<String>();
allTableNames.add(tableName);
IdealState idealState = new IdealState(tableName);
idealState.setPartitionState("myTable_0", "pinot1", "OFFLINE");
idealState.setPartitionState("myTable_0", "pinot2", "OFFLINE");
idealState.setPartitionState("myTable_0", "pinot3", "OFFLINE");
idealState.setReplicas("0");
idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED);
{
helixResourceManager = mock(PinotHelixResourceManager.class);
when(helixResourceManager.getAllTables()).thenReturn(allTableNames);
when(helixResourceManager.getTableIdealState(tableName)).thenReturn(idealState);
when(helixResourceManager.getTableExternalView(tableName)).thenReturn(null);
}
{
config = mock(ControllerConf.class);
when(config.getStatusCheckerFrequencyInSeconds()).thenReturn(300);
when(config.getStatusCheckerWaitForPushTimeInSeconds()).thenReturn(300);
}
{
leadControllerManager = mock(LeadControllerManager.class);
when(leadControllerManager.isLeaderForTable(anyString())).thenReturn(true);
}
metricsRegistry = new MetricsRegistry();
controllerMetrics = new ControllerMetrics(metricsRegistry);
segmentStatusChecker =
new SegmentStatusChecker(helixResourceManager, leadControllerManager, config, controllerMetrics);
segmentStatusChecker.start();
segmentStatusChecker.run();
Assert.assertEquals(controllerMetrics.getValueOfTableGauge(tableName, ControllerGauge.SEGMENTS_IN_ERROR_STATE), 0);
Assert.assertEquals(controllerMetrics.getValueOfTableGauge(tableName, ControllerGauge.NUMBER_OF_REPLICAS), 1);
Assert.assertEquals(controllerMetrics.getValueOfTableGauge(tableName, ControllerGauge.PERCENT_OF_REPLICAS), 100);
Assert.assertEquals(controllerMetrics.getValueOfTableGauge(tableName, ControllerGauge.PERCENT_SEGMENTS_AVAILABLE),
100);
}
Example 5
| Source File: HelixMirrorMakerManager.java From uReplicator with Apache License 2.0 | 4 votes |
|
public synchronized void updateTopicPartitionStateInMirrorMaker(String topicName, int partition,
String state) {
updateCurrentServingInstance();
if (!Constants.HELIX_OFFLINE_STATE.equalsIgnoreCase(state) && !Constants.HELIX_ONLINE_STATE
.equalsIgnoreCase(state)) {
throw new IllegalArgumentException(
String.format("Failed to update topic %s, partition %d to invalid state %s.",
topicName, partition, state));
}
IdealState idealState = _helixAdmin.getResourceIdealState(_helixClusterName, topicName);
String partitionName = String.valueOf(partition);
if (idealState == null ||
partition >= idealState.getNumPartitions()) {
throw new IllegalArgumentException(
String.format("Topic %s, partition %d not exists in current route.",
topicName, partition));
}
String instanceName;
if (idealState.getInstanceStateMap(partitionName).keySet().isEmpty()) {
if (Constants.HELIX_OFFLINE_STATE.equalsIgnoreCase(state)) {
throw new IllegalArgumentException(
String.format("Topic %s, partition %d not exists in current route.",
topicName, partition));
} else if (_currentServingInstance.isEmpty()) {
throw new InternalError("No available worker");
}
instanceName = _currentServingInstance.poll().getInstanceName();
} else {
instanceName = idealState.getInstanceStateMap(partitionName).keySet().iterator().next();
String oldState = idealState.getInstanceStateMap(partitionName).get(instanceName);
if (oldState.equalsIgnoreCase(state)) {
throw new IllegalArgumentException(String.format("Topic %s, partition %d already set %s",
idealState.getResourceName(), partition, state));
}
}
idealState.setPartitionState(partitionName, instanceName, state);
_helixAdmin.setResourceIdealState(_helixClusterName, topicName, idealState);
}
Example 6
| Source File: DefaultSchedulerMessageHandlerFactory.java From helix with Apache License 2.0 | 4 votes |
|
void handleMessageUsingScheduledTaskQueue(Criteria recipientCriteria, Message messageTemplate,
String controllerMsgId) {
HelixDataAccessor accessor = _manager.getHelixDataAccessor();
Builder keyBuilder = accessor.keyBuilder();
String clusterName = recipientCriteria.getClusterName();
if (clusterName != null && !clusterName.equals(_manager.getClusterName())) {
throw new HelixException(String.format(
"ScheduledTaskQueue cannot send message to another cluster. Local cluster name %s, remote cluster name %s.",
_manager.getClusterName(), clusterName));
}
Map<String, String> sendSummary = new HashMap<String, String>();
sendSummary.put("MessageCount", "0");
Map<InstanceType, List<Message>> messages =
_manager.getMessagingService().generateMessage(recipientCriteria, messageTemplate);
// Calculate tasks, and put them into the idealState of the SCHEDULER_TASK_QUEUE resource.
// List field are the destination node, while the Message parameters are stored in the
// mapFields
// task throttling can be done on SCHEDULER_TASK_QUEUE resource
if (messages.size() > 0) {
String taskQueueName = _message.getRecord().getSimpleField(SCHEDULER_TASK_QUEUE);
if (taskQueueName == null) {
throw new HelixException("SchedulerTaskMessage need to have " + SCHEDULER_TASK_QUEUE
+ " specified.");
}
IdealState newAddedScheduledTasks = new IdealState(taskQueueName);
newAddedScheduledTasks.setBucketSize(TASKQUEUE_BUCKET_NUM);
newAddedScheduledTasks.setStateModelDefRef(SCHEDULER_TASK_QUEUE);
synchronized (_manager) {
int existingTopPartitionId = 0;
IdealState currentTaskQueue =
_manager.getHelixDataAccessor().getProperty(
accessor.keyBuilder().idealStates(newAddedScheduledTasks.getId()));
if (currentTaskQueue != null) {
existingTopPartitionId = findTopPartitionId(currentTaskQueue) + 1;
}
List<Message> taskMessages = (List<Message>) (messages.values().toArray()[0]);
for (Message task : taskMessages) {
String partitionId = taskQueueName + "_" + existingTopPartitionId;
existingTopPartitionId++;
String instanceName = task.getTgtName();
newAddedScheduledTasks.setPartitionState(partitionId, instanceName, "COMPLETED");
task.getRecord().setSimpleField(instanceName, "COMPLETED");
task.getRecord().setSimpleField(CONTROLLER_MSG_ID, controllerMsgId);
List<String> priorityList = new LinkedList<String>();
priorityList.add(instanceName);
newAddedScheduledTasks.getRecord().setListField(partitionId, priorityList);
newAddedScheduledTasks.getRecord().setMapField(partitionId,
task.getRecord().getSimpleFields());
_logger.info("Scheduling for controllerMsg " + controllerMsgId + " , sending task "
+ partitionId + " " + task.getMsgId() + " to " + instanceName);
if (_logger.isDebugEnabled()) {
_logger.debug(task.getRecord().getSimpleFields().toString());
}
}
_manager.getHelixDataAccessor().updateProperty(
accessor.keyBuilder().idealStates(newAddedScheduledTasks.getId()),
newAddedScheduledTasks);
sendSummary.put("MessageCount", "" + taskMessages.size());
}
}
// Record the number of messages sent into scheduler message status updates
ZNRecord statusUpdate =
accessor.getProperty(
keyBuilder.controllerTaskStatus(MessageType.SCHEDULER_MSG.name(),
_message.getMsgId())).getRecord();
statusUpdate.getMapFields().put("SentMessageCount", sendSummary);
accessor.updateProperty(keyBuilder.controllerTaskStatus(MessageType.SCHEDULER_MSG.name(),
_message.getMsgId()), new StatusUpdate(statusUpdate));
}
Example 7
| Source File: PinotTableIdealStateBuilder.java From incubator-pinot with Apache License 2.0 | 4 votes |
|
public static IdealState addNewRealtimeSegmentToIdealState(String segmentId, IdealState state, String instanceName) {
state.setPartitionState(segmentId, instanceName, ONLINE);
state.setNumPartitions(state.getNumPartitions() + 1);
return state;
}
Example 8
| Source File: SegmentStatusCheckerTest.java From incubator-pinot with Apache License 2.0 | 4 votes |
|
@Test
public void offlineBasicTest()
throws Exception {
final String tableName = "myTable_OFFLINE";
List<String> allTableNames = new ArrayList<String>();
allTableNames.add(tableName);
IdealState idealState = new IdealState(tableName);
idealState.setPartitionState("myTable_0", "pinot1", "ONLINE");
idealState.setPartitionState("myTable_0", "pinot2", "ONLINE");
idealState.setPartitionState("myTable_0", "pinot3", "ONLINE");
idealState.setPartitionState("myTable_1", "pinot1", "ONLINE");
idealState.setPartitionState("myTable_1", "pinot2", "ONLINE");
idealState.setPartitionState("myTable_1", "pinot3", "ONLINE");
idealState.setPartitionState("myTable_2", "pinot3", "OFFLINE");
idealState.setReplicas("2");
idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED);
ExternalView externalView = new ExternalView(tableName);
externalView.setState("myTable_0", "pinot1", "ONLINE");
externalView.setState("myTable_0", "pinot2", "ONLINE");
externalView.setState("myTable_1", "pinot1", "ERROR");
externalView.setState("myTable_1", "pinot2", "ONLINE");
{
helixResourceManager = mock(PinotHelixResourceManager.class);
when(helixResourceManager.getAllTables()).thenReturn(allTableNames);
when(helixResourceManager.getTableIdealState(tableName)).thenReturn(idealState);
when(helixResourceManager.getTableExternalView(tableName)).thenReturn(externalView);
}
{
config = mock(ControllerConf.class);
when(config.getStatusCheckerFrequencyInSeconds()).thenReturn(300);
when(config.getStatusCheckerWaitForPushTimeInSeconds()).thenReturn(300);
}
{
leadControllerManager = mock(LeadControllerManager.class);
when(leadControllerManager.isLeaderForTable(anyString())).thenReturn(true);
}
metricsRegistry = new MetricsRegistry();
controllerMetrics = new ControllerMetrics(metricsRegistry);
segmentStatusChecker =
new SegmentStatusChecker(helixResourceManager, leadControllerManager, config, controllerMetrics);
segmentStatusChecker.start();
segmentStatusChecker.run();
Assert.assertEquals(
controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.SEGMENTS_IN_ERROR_STATE), 1);
Assert
.assertEquals(controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.NUMBER_OF_REPLICAS),
1);
Assert
.assertEquals(controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.PERCENT_OF_REPLICAS),
33);
Assert.assertEquals(
controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.PERCENT_SEGMENTS_AVAILABLE), 100);
}
Example 9
| Source File: SegmentStatusCheckerTest.java From incubator-pinot with Apache License 2.0 | 4 votes |
|
@Test
public void realtimeBasicTest()
throws Exception {
final String tableName = "myTable_REALTIME";
final String rawTableName = TableNameBuilder.extractRawTableName(tableName);
List<String> allTableNames = new ArrayList<String>();
allTableNames.add(tableName);
final LLCSegmentName seg1 = new LLCSegmentName(rawTableName, 1, 0, System.currentTimeMillis());
final LLCSegmentName seg2 = new LLCSegmentName(rawTableName, 1, 1, System.currentTimeMillis());
final LLCSegmentName seg3 = new LLCSegmentName(rawTableName, 2, 1, System.currentTimeMillis());
IdealState idealState = new IdealState(tableName);
idealState.setPartitionState(seg1.getSegmentName(), "pinot1", "ONLINE");
idealState.setPartitionState(seg1.getSegmentName(), "pinot2", "ONLINE");
idealState.setPartitionState(seg1.getSegmentName(), "pinot3", "ONLINE");
idealState.setPartitionState(seg2.getSegmentName(), "pinot1", "ONLINE");
idealState.setPartitionState(seg2.getSegmentName(), "pinot2", "ONLINE");
idealState.setPartitionState(seg2.getSegmentName(), "pinot3", "ONLINE");
idealState.setPartitionState(seg3.getSegmentName(), "pinot1", "CONSUMING");
idealState.setPartitionState(seg3.getSegmentName(), "pinot2", "CONSUMING");
idealState.setPartitionState(seg3.getSegmentName(), "pinot3", "OFFLINE");
idealState.setReplicas("3");
idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED);
ExternalView externalView = new ExternalView(tableName);
externalView.setState(seg1.getSegmentName(), "pinot1", "ONLINE");
externalView.setState(seg1.getSegmentName(), "pinot2", "ONLINE");
externalView.setState(seg1.getSegmentName(), "pinot3", "ONLINE");
externalView.setState(seg2.getSegmentName(), "pinot1", "CONSUMING");
externalView.setState(seg2.getSegmentName(), "pinot2", "ONLINE");
externalView.setState(seg2.getSegmentName(), "pinot3", "CONSUMING");
externalView.setState(seg3.getSegmentName(), "pinot1", "CONSUMING");
externalView.setState(seg3.getSegmentName(), "pinot2", "CONSUMING");
externalView.setState(seg3.getSegmentName(), "pinot3", "OFFLINE");
{
helixResourceManager = mock(PinotHelixResourceManager.class);
when(helixResourceManager.getAllTables()).thenReturn(allTableNames);
when(helixResourceManager.getTableIdealState(tableName)).thenReturn(idealState);
when(helixResourceManager.getTableExternalView(tableName)).thenReturn(externalView);
}
{
config = mock(ControllerConf.class);
when(config.getStatusCheckerFrequencyInSeconds()).thenReturn(300);
when(config.getStatusCheckerWaitForPushTimeInSeconds()).thenReturn(300);
}
{
leadControllerManager = mock(LeadControllerManager.class);
when(leadControllerManager.isLeaderForTable(anyString())).thenReturn(true);
}
metricsRegistry = new MetricsRegistry();
controllerMetrics = new ControllerMetrics(metricsRegistry);
segmentStatusChecker =
new SegmentStatusChecker(helixResourceManager, leadControllerManager, config, controllerMetrics);
segmentStatusChecker.start();
segmentStatusChecker.run();
Assert.assertEquals(
controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.SEGMENTS_IN_ERROR_STATE), 0);
Assert
.assertEquals(controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.NUMBER_OF_REPLICAS),
3);
Assert
.assertEquals(controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.PERCENT_OF_REPLICAS),
100);
Assert.assertEquals(
controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.PERCENT_SEGMENTS_AVAILABLE), 100);
}
Example 10
| Source File: SegmentStatusCheckerTest.java From incubator-pinot with Apache License 2.0 | 4 votes |
|
@Test
public void missingEVPartitionTest()
throws Exception {
final String tableName = "myTable_OFFLINE";
List<String> allTableNames = new ArrayList<String>();
allTableNames.add(tableName);
IdealState idealState = new IdealState(tableName);
idealState.setPartitionState("myTable_0", "pinot1", "ONLINE");
idealState.setPartitionState("myTable_0", "pinot2", "ONLINE");
idealState.setPartitionState("myTable_0", "pinot3", "ONLINE");
idealState.setPartitionState("myTable_1", "pinot1", "ONLINE");
idealState.setPartitionState("myTable_1", "pinot2", "ONLINE");
idealState.setPartitionState("myTable_1", "pinot3", "ONLINE");
idealState.setPartitionState("myTable_2", "pinot3", "OFFLINE");
idealState.setPartitionState("myTable_3", "pinot3", "ONLINE");
idealState.setReplicas("2");
idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED);
ExternalView externalView = new ExternalView(tableName);
externalView.setState("myTable_0", "pinot1", "ONLINE");
externalView.setState("myTable_0", "pinot2", "ONLINE");
externalView.setState("myTable_1", "pinot1", "ERROR");
externalView.setState("myTable_1", "pinot2", "ONLINE");
ZNRecord znrecord = new ZNRecord("myTable_0");
znrecord.setSimpleField(CommonConstants.Segment.SEGMENT_NAME, "myTable_0");
znrecord.setSimpleField(CommonConstants.Segment.TABLE_NAME, "myTable_OFFLINE");
znrecord.setSimpleField(CommonConstants.Segment.INDEX_VERSION, "v1");
znrecord.setEnumField(CommonConstants.Segment.SEGMENT_TYPE, CommonConstants.Segment.SegmentType.OFFLINE);
znrecord.setLongField(CommonConstants.Segment.START_TIME, 1000);
znrecord.setLongField(CommonConstants.Segment.END_TIME, 2000);
znrecord.setSimpleField(CommonConstants.Segment.TIME_UNIT, TimeUnit.HOURS.toString());
znrecord.setLongField(CommonConstants.Segment.TOTAL_DOCS, 10000);
znrecord.setLongField(CommonConstants.Segment.CRC, 1234);
znrecord.setLongField(CommonConstants.Segment.CREATION_TIME, 3000);
znrecord.setSimpleField(CommonConstants.Segment.Offline.DOWNLOAD_URL, "http://localhost:8000/myTable_0");
znrecord.setLongField(CommonConstants.Segment.Offline.PUSH_TIME, System.currentTimeMillis());
znrecord.setLongField(CommonConstants.Segment.Offline.REFRESH_TIME, System.currentTimeMillis());
ZkHelixPropertyStore<ZNRecord> propertyStore;
{
propertyStore = (ZkHelixPropertyStore<ZNRecord>) mock(ZkHelixPropertyStore.class);
when(propertyStore.get("/SEGMENTS/myTable_OFFLINE/myTable_3", null, AccessOption.PERSISTENT))
.thenReturn(znrecord);
}
{
helixResourceManager = mock(PinotHelixResourceManager.class);
when(helixResourceManager.getAllTables()).thenReturn(allTableNames);
when(helixResourceManager.getTableIdealState(tableName)).thenReturn(idealState);
when(helixResourceManager.getTableExternalView(tableName)).thenReturn(externalView);
when(helixResourceManager.getOfflineSegmentZKMetadata(tableName, "myTable_3"))
.thenReturn(new OfflineSegmentZKMetadata(znrecord));
}
{
config = mock(ControllerConf.class);
when(config.getStatusCheckerFrequencyInSeconds()).thenReturn(300);
when(config.getStatusCheckerWaitForPushTimeInSeconds()).thenReturn(0);
}
{
leadControllerManager = mock(LeadControllerManager.class);
when(leadControllerManager.isLeaderForTable(anyString())).thenReturn(true);
}
metricsRegistry = new MetricsRegistry();
controllerMetrics = new ControllerMetrics(metricsRegistry);
segmentStatusChecker =
new SegmentStatusChecker(helixResourceManager, leadControllerManager, config, controllerMetrics);
segmentStatusChecker.start();
segmentStatusChecker.run();
Assert.assertEquals(
controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.SEGMENTS_IN_ERROR_STATE), 1);
Assert
.assertEquals(controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.NUMBER_OF_REPLICAS),
0);
Assert.assertEquals(
controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.PERCENT_SEGMENTS_AVAILABLE), 75);
}
Example 11
| Source File: SegmentStatusCheckerTest.java From incubator-pinot with Apache License 2.0 | 4 votes |
|
@Test
public void missingEVTest()
throws Exception {
final String tableName = "myTable_REALTIME";
List<String> allTableNames = new ArrayList<String>();
allTableNames.add(tableName);
IdealState idealState = new IdealState(tableName);
idealState.setPartitionState("myTable_0", "pinot1", "ONLINE");
idealState.setPartitionState("myTable_0", "pinot2", "ONLINE");
idealState.setPartitionState("myTable_0", "pinot3", "ONLINE");
idealState.setPartitionState("myTable_1", "pinot1", "ONLINE");
idealState.setPartitionState("myTable_1", "pinot2", "ONLINE");
idealState.setPartitionState("myTable_1", "pinot3", "ONLINE");
idealState.setPartitionState("myTable_2", "pinot3", "OFFLINE");
idealState.setReplicas("2");
idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED);
{
helixResourceManager = mock(PinotHelixResourceManager.class);
when(helixResourceManager.getAllTables()).thenReturn(allTableNames);
when(helixResourceManager.getTableIdealState(tableName)).thenReturn(idealState);
when(helixResourceManager.getTableExternalView(tableName)).thenReturn(null);
}
{
config = mock(ControllerConf.class);
when(config.getStatusCheckerFrequencyInSeconds()).thenReturn(300);
when(config.getStatusCheckerWaitForPushTimeInSeconds()).thenReturn(300);
}
{
leadControllerManager = mock(LeadControllerManager.class);
when(leadControllerManager.isLeaderForTable(anyString())).thenReturn(true);
}
metricsRegistry = new MetricsRegistry();
controllerMetrics = new ControllerMetrics(metricsRegistry);
segmentStatusChecker =
new SegmentStatusChecker(helixResourceManager, leadControllerManager, config, controllerMetrics);
segmentStatusChecker.start();
segmentStatusChecker.run();
Assert.assertEquals(controllerMetrics.getValueOfTableGauge(tableName, ControllerGauge.SEGMENTS_IN_ERROR_STATE), 0);
Assert.assertEquals(controllerMetrics.getValueOfTableGauge(tableName, ControllerGauge.NUMBER_OF_REPLICAS), 0);
}
Example 12
| Source File: SegmentStatusCheckerTest.java From incubator-pinot with Apache License 2.0 | 4 votes |
|
@Test
public void missingEVPartitionPushTest()
throws Exception {
final String tableName = "myTable_OFFLINE";
List<String> allTableNames = new ArrayList<String>();
allTableNames.add(tableName);
IdealState idealState = new IdealState(tableName);
idealState.setPartitionState("myTable_0", "pinot1", "ONLINE");
idealState.setPartitionState("myTable_1", "pinot1", "ONLINE");
idealState.setPartitionState("myTable_1", "pinot2", "ONLINE");
idealState.setReplicas("2");
idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED);
ExternalView externalView = new ExternalView(tableName);
externalView.setState("myTable_1", "pinot1", "ONLINE");
externalView.setState("myTable_1", "pinot2", "ONLINE");
ZNRecord znrecord = new ZNRecord("myTable_0");
znrecord.setSimpleField(CommonConstants.Segment.SEGMENT_NAME, "myTable_0");
znrecord.setSimpleField(CommonConstants.Segment.TABLE_NAME, "myTable_OFFLINE");
znrecord.setSimpleField(CommonConstants.Segment.INDEX_VERSION, "v1");
znrecord.setEnumField(CommonConstants.Segment.SEGMENT_TYPE, CommonConstants.Segment.SegmentType.OFFLINE);
znrecord.setLongField(CommonConstants.Segment.START_TIME, 1000);
znrecord.setLongField(CommonConstants.Segment.END_TIME, 2000);
znrecord.setSimpleField(CommonConstants.Segment.TIME_UNIT, TimeUnit.HOURS.toString());
znrecord.setLongField(CommonConstants.Segment.TOTAL_DOCS, 10000);
znrecord.setLongField(CommonConstants.Segment.CRC, 1234);
znrecord.setLongField(CommonConstants.Segment.CREATION_TIME, 3000);
znrecord.setSimpleField(CommonConstants.Segment.Offline.DOWNLOAD_URL, "http://localhost:8000/myTable_0");
znrecord.setLongField(CommonConstants.Segment.Offline.PUSH_TIME, System.currentTimeMillis());
znrecord.setLongField(CommonConstants.Segment.Offline.REFRESH_TIME, System.currentTimeMillis());
{
helixResourceManager = mock(PinotHelixResourceManager.class);
when(helixResourceManager.getAllTables()).thenReturn(allTableNames);
when(helixResourceManager.getTableIdealState(tableName)).thenReturn(idealState);
when(helixResourceManager.getTableExternalView(tableName)).thenReturn(externalView);
when(helixResourceManager.getOfflineSegmentZKMetadata(tableName, "myTable_0"))
.thenReturn(new OfflineSegmentZKMetadata(znrecord));
}
{
config = mock(ControllerConf.class);
when(config.getStatusCheckerFrequencyInSeconds()).thenReturn(300);
when(config.getStatusCheckerWaitForPushTimeInSeconds()).thenReturn(300);
}
{
leadControllerManager = mock(LeadControllerManager.class);
when(leadControllerManager.isLeaderForTable(anyString())).thenReturn(true);
}
metricsRegistry = new MetricsRegistry();
controllerMetrics = new ControllerMetrics(metricsRegistry);
segmentStatusChecker =
new SegmentStatusChecker(helixResourceManager, leadControllerManager, config, controllerMetrics);
segmentStatusChecker.start();
segmentStatusChecker.run();
Assert.assertEquals(
controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.SEGMENTS_IN_ERROR_STATE), 0);
Assert
.assertEquals(controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.NUMBER_OF_REPLICAS),
2);
Assert
.assertEquals(controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.PERCENT_OF_REPLICAS),
100);
Assert.assertEquals(
controllerMetrics.getValueOfTableGauge(externalView.getId(), ControllerGauge.PERCENT_SEGMENTS_AVAILABLE), 100);
}
Example 13
| Source File: SegmentStatusCheckerTest.java From incubator-pinot with Apache License 2.0 | 4 votes |
|
@Test
public void disabledTableTest()
throws Exception {
final String tableName = "myTable_OFFLINE";
List<String> allTableNames = new ArrayList<String>();
allTableNames.add(tableName);
IdealState idealState = new IdealState(tableName);
// disable table in idealstate
idealState.enable(false);
idealState.setPartitionState("myTable_OFFLINE", "pinot1", "OFFLINE");
idealState.setPartitionState("myTable_OFFLINE", "pinot2", "OFFLINE");
idealState.setPartitionState("myTable_OFFLINE", "pinot3", "OFFLINE");
idealState.setReplicas("1");
idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED);
{
helixResourceManager = mock(PinotHelixResourceManager.class);
when(helixResourceManager.getAllTables()).thenReturn(allTableNames);
when(helixResourceManager.getTableIdealState(tableName)).thenReturn(idealState);
when(helixResourceManager.getTableExternalView(tableName)).thenReturn(null);
}
{
config = mock(ControllerConf.class);
when(config.getStatusCheckerFrequencyInSeconds()).thenReturn(300);
when(config.getStatusCheckerWaitForPushTimeInSeconds()).thenReturn(300);
}
{
leadControllerManager = mock(LeadControllerManager.class);
when(leadControllerManager.isLeaderForTable(anyString())).thenReturn(true);
}
metricsRegistry = new MetricsRegistry();
controllerMetrics = new ControllerMetrics(metricsRegistry);
segmentStatusChecker =
new SegmentStatusChecker(helixResourceManager, leadControllerManager, config, controllerMetrics);
// verify state before test
Assert.assertEquals(controllerMetrics.getValueOfGlobalGauge(ControllerGauge.DISABLED_TABLE_COUNT), 0);
// update metrics
segmentStatusChecker.start();
segmentStatusChecker.run();
Assert.assertEquals(controllerMetrics.getValueOfGlobalGauge(ControllerGauge.DISABLED_TABLE_COUNT), 1);
}
Example 14
| Source File: RetentionManagerTest.java From incubator-pinot with Apache License 2.0 | 4 votes |
|
private PinotHelixResourceManager setupSegmentMetadata(TableConfig tableConfig, final long now, final int nSegments,
List<String> segmentsToBeDeleted) {
final int replicaCount = Integer.valueOf(tableConfig.getValidationConfig().getReplicasPerPartition());
List<RealtimeSegmentZKMetadata> allSegments = new ArrayList<>();
IdealState idealState =
PinotTableIdealStateBuilder.buildEmptyRealtimeIdealStateFor(REALTIME_TABLE_NAME, replicaCount, true);
final int kafkaPartition = 5;
final long millisInDays = TimeUnit.DAYS.toMillis(1);
final String serverName = "Server_localhost_0";
// If we set the segment creation time to a certain value and compare it as being X ms old,
// then we could get unpredictable results depending on whether it takes more or less than
// one millisecond to get to RetentionManager time comparison code. To be safe, set the
// milliseconds off by 1/2 day.
long segmentCreationTime = now - (nSegments + 1) * millisInDays + millisInDays / 2;
for (int seq = 1; seq <= nSegments; seq++) {
segmentCreationTime += millisInDays;
LLCRealtimeSegmentZKMetadata segmentMetadata = createSegmentMetadata(replicaCount, segmentCreationTime);
LLCSegmentName llcSegmentName = new LLCSegmentName(TEST_TABLE_NAME, kafkaPartition, seq, segmentCreationTime);
final String segName = llcSegmentName.getSegmentName();
segmentMetadata.setSegmentName(segName);
if (seq == nSegments) {
// create consuming segment
segmentMetadata.setStatus(CommonConstants.Segment.Realtime.Status.IN_PROGRESS);
idealState.setPartitionState(segName, serverName, "CONSUMING");
allSegments.add(segmentMetadata);
} else if (seq == 1) {
// create IN_PROGRESS metadata absent from ideal state, older than 5 days
segmentMetadata.setStatus(CommonConstants.Segment.Realtime.Status.IN_PROGRESS);
allSegments.add(segmentMetadata);
segmentsToBeDeleted.add(segmentMetadata.getSegmentName());
} else if (seq == nSegments - 1) {
// create IN_PROGRESS metadata absent from ideal state, younger than 5 days
segmentMetadata.setStatus(CommonConstants.Segment.Realtime.Status.IN_PROGRESS);
allSegments.add(segmentMetadata);
} else if (seq % 2 == 0) {
// create ONLINE segment
segmentMetadata.setStatus(CommonConstants.Segment.Realtime.Status.DONE);
idealState.setPartitionState(segName, serverName, "ONLINE");
allSegments.add(segmentMetadata);
} else {
segmentMetadata.setStatus(CommonConstants.Segment.Realtime.Status.IN_PROGRESS);
idealState.setPartitionState(segName, serverName, "OFFLINE");
allSegments.add(segmentMetadata);
if (now - segmentCreationTime > RetentionManager.OLD_LLC_SEGMENTS_RETENTION_IN_MILLIS) {
segmentsToBeDeleted.add(segmentMetadata.getSegmentName());
}
}
}
PinotHelixResourceManager pinotHelixResourceManager = mock(PinotHelixResourceManager.class);
when(pinotHelixResourceManager.getTableConfig(REALTIME_TABLE_NAME)).thenReturn(tableConfig);
when(pinotHelixResourceManager.getRealtimeSegmentMetadata(REALTIME_TABLE_NAME)).thenReturn(allSegments);
when(pinotHelixResourceManager.getHelixClusterName()).thenReturn(HELIX_CLUSTER_NAME);
HelixAdmin helixAdmin = mock(HelixAdmin.class);
when(helixAdmin.getResourceIdealState(HELIX_CLUSTER_NAME, REALTIME_TABLE_NAME)).thenReturn(idealState);
when(pinotHelixResourceManager.getHelixAdmin()).thenReturn(helixAdmin);
return pinotHelixResourceManager;
}
Example 15
| Source File: ServiceStatusTest.java From incubator-pinot with Apache License 2.0 | 4 votes |
|
private void testMultipleResourcesAndPercent(double percentReady) {
final long now = System.currentTimeMillis();
random = new Random(now);
final String clusterName = "noSuchCluster";
final List<String> tables = new ArrayList<>();
final int tableCount = 2500 + random.nextInt(100);
int readyTables = 0;
Map<String, IdealState> idealStates = new HashMap<>();
Map<String, ExternalView> externalViews = new HashMap<>();
for (int i = 1; i <= tableCount; i++) {
final String tableName = generateRandomString(10) + String.valueOf(i);
tables.add(tableName);
final String segmentName = "segment1";
String evState;
if (random.nextDouble() * 100 < percentReady) {
evState = "ONLINE";
readyTables++;
} else {
evState = "OFFLINE";
}
IdealState idealState = new IdealState(tableName);
idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED);
idealState.setPartitionState(segmentName, INSTANCE_NAME, "ONLINE");
ExternalView externalView = new ExternalView(tableName);
externalView.setState(segmentName, INSTANCE_NAME, evState);
idealStates.put(tableName, idealState);
externalViews.put(tableName, externalView);
}
final double actualReadyPercent = (double) readyTables * 100 / tableCount;
double lowestReadyPercent = (int) Math.round(actualReadyPercent);
lowestReadyPercent = lowestReadyPercent > 2 ? lowestReadyPercent - 2 : 1; // Should be 2 below percentReady
// Create ServiceCallback objects with minReadyPercent set to values between lowestReadyPercent and 100.
// Call getServiceStatus() enough number of times so that we are only left with the tables
// that are not ready yet. We need to call getServiceStatus() at most tableCount times.
for (double minReadyPercent = lowestReadyPercent; minReadyPercent <= 100; minReadyPercent += 0.1) {
TestMultiResourceISAndEVMatchCB callback =
new TestMultiResourceISAndEVMatchCB(clusterName, INSTANCE_NAME, tables, minReadyPercent);
callback.setIdealStates(idealStates);
callback.setExternalViews(externalViews);
ServiceStatus.Status status = callback.getServiceStatus();
// we need to call getServiceStatus() at most the number of bad tables plus 1,
// to get a STARTED condition if we can. After that, the return value should
// never change.
final int nBadTables = tableCount - readyTables;
for (int i = 0; i <= nBadTables; i++) {
status = callback.getServiceStatus();
}
ServiceStatus.Status expected =
minReadyPercent > actualReadyPercent ? ServiceStatus.Status.STARTING : ServiceStatus.Status.GOOD;
String errorMsg = "Mismatch at " + minReadyPercent + "%, tableCount=" + tableCount + ", percentTablesReady="
+ actualReadyPercent + ":" + callback.getStatusDescription();
Assert.assertEquals(status, expected, errorMsg);
// The status should never change going forward from here.
for (int i = nBadTables + 1; i < tableCount; i++) {
ServiceStatus.Status laterStatus = callback.getServiceStatus();
String msg = "Mismatch at " + minReadyPercent + "%, tableCount=" + tableCount + ", percentTablesReady="
+ actualReadyPercent + ", i=" + i + ":" + callback.getStatusDescription();
Assert.assertEquals(laterStatus, status, msg);
}
}
}
