Java Code Examples for org.apache.kafka.common.Cluster#partition()

/**
 * @return True if the metadata has changed, false otherwise.
 */
public static boolean metadataChanged(Cluster prev, Cluster curr) {
  // Broker has changed.
  Set<Node> prevNodeSet = new HashSet<>(prev.nodes());
  if (prevNodeSet.size() != curr.nodes().size()) {
    return true;
  }
  prevNodeSet.removeAll(curr.nodes());
  if (!prevNodeSet.isEmpty()) {
    return true;
  }
  // Topic has changed
  if (!prev.topics().equals(curr.topics())) {
    return true;
  }

  // partition has changed.
  for (String topic : prev.topics()) {
    if (!prev.partitionCountForTopic(topic).equals(curr.partitionCountForTopic(topic))) {
      return true;
    }
    for (PartitionInfo prevPartInfo : prev.partitionsForTopic(topic)) {
      PartitionInfo currPartInfo = curr.partition(new TopicPartition(prevPartInfo.topic(), prevPartInfo.partition()));
      if (leaderChanged(prevPartInfo, currPartInfo) || replicaListChanged(prevPartInfo, currPartInfo)) {
        return true;
      }
    }
  }
  return false;
}
 

/**
 * For each proposal, create a replica action task if there is a need for moving replica(s) between brokers to reach
 * expected final proposal state.
 *
 * @param proposals Execution proposals.
 * @param cluster Kafka cluster state.
 * @param replicaMovementStrategy The strategy used to determine the execution order of generated replica movement tasks.
 */
private void maybeAddInterBrokerReplicaMovementTasks(Collection<ExecutionProposal> proposals,
                                                     Cluster cluster,
                                                     ReplicaMovementStrategy replicaMovementStrategy) {
  for (ExecutionProposal proposal : proposals) {
    TopicPartition tp = proposal.topicPartition();
    PartitionInfo partitionInfo = cluster.partition(tp);
    if (partitionInfo == null) {
      LOG.trace("Ignored the attempt to move non-existing partition for topic partition: {}", tp);
      continue;
    }
    if (!proposal.isInterBrokerMovementCompleted(partitionInfo)) {
      long replicaActionExecutionId = _executionId++;
      long executionAlertingThresholdMs = Math.max(Math.round(proposal.dataToMoveInMB() / _interBrokerReplicaMovementRateAlertingThreshold),
                                                   _taskExecutionAlertingThresholdMs);
      ExecutionTask executionTask = new ExecutionTask(replicaActionExecutionId, proposal, INTER_BROKER_REPLICA_ACTION,
                                                      executionAlertingThresholdMs);
      _remainingInterBrokerReplicaMovements.add(executionTask);
      LOG.trace("Added action {} as replica proposal {}", replicaActionExecutionId, proposal);
    }
  }
  if (replicaMovementStrategy == null) {
    _interPartMoveTaskByBrokerId = _defaultReplicaMovementTaskStrategy.applyStrategy(_remainingInterBrokerReplicaMovements,
                                                                                     cluster);
  } else {
    // Chain the generated composite strategy with BaseReplicaMovementStrategy in the end to ensure the returned
    // strategy can always determine the order of two tasks.
    _interPartMoveTaskByBrokerId = replicaMovementStrategy.chain(new BaseReplicaMovementStrategy())
            .applyStrategy(_remainingInterBrokerReplicaMovements, cluster);
  }
}
 

/**
 * For a inter-broker replica movement action, the completion depends on the task state:
 * IN_PROGRESS: when the current replica list is the same as the new replica list and all replicas are in-sync.
 * ABORTING: done when the current replica list is the same as the old replica list. Due to race condition,
 *           we also consider it done if the current replica list is the same as the new replica list and all replicas
 *           are in-sync.
 * DEAD: always considered as done because we neither move forward or rollback.
 *
 * There should be no other task state seen here.
 */
private boolean isInterBrokerReplicaActionDone(Cluster cluster, TopicPartition tp, ExecutionTask task) {
  PartitionInfo partitionInfo = cluster.partition(tp);
  switch (task.state()) {
    case IN_PROGRESS:
      return task.proposal().isInterBrokerMovementCompleted(partitionInfo);
    case ABORTING:
      return task.proposal().isInterBrokerMovementAborted(partitionInfo);
    case DEAD:
      return true;
    default:
      throw new IllegalStateException("Should never be here. State " + task.state());
  }
}
 

/**
 * Create replicas of the partition with the given (1) identifier and (2) load information to populate the given cluster model.
 * If partition with the given identifier does not exist in the given cluster, do nothing.
 *
 * @param cluster Kafka cluster.
 * @param clusterModel The cluster model to populate load information.
 * @param tp Topic partition that identifies the partition to populate the load for.
 * @param valuesAndExtrapolations The values and extrapolations of the leader replica.
 * @param replicaPlacementInfo The distribution of replicas over broker logdirs if available, {@code null} otherwise.
 * @param brokerCapacityConfigResolver The resolver for retrieving broker capacities.
 * @param allowCapacityEstimation whether allow capacity estimation in cluster model if the underlying live broker capacity is unavailable.
 */
static void populatePartitionLoad(Cluster cluster,
                                  ClusterModel clusterModel,
                                  TopicPartition tp,
                                  ValuesAndExtrapolations valuesAndExtrapolations,
                                  Map<TopicPartition, Map<Integer, String>> replicaPlacementInfo,
                                  BrokerCapacityConfigResolver brokerCapacityConfigResolver,
                                  boolean allowCapacityEstimation)
    throws TimeoutException {
  PartitionInfo partitionInfo = cluster.partition(tp);
  // If partition info does not exist, the topic may have been deleted.
  if (partitionInfo != null) {
    Set<Integer> aliveBrokers = cluster.nodes().stream().mapToInt(Node::id).boxed().collect(Collectors.toSet());
    boolean needToAdjustCpuUsage = true;
    Set<Integer> deadBrokersWithUnknownCapacity = new HashSet<>();
    for (int index = 0; index < partitionInfo.replicas().length; index++) {
      Node replica = partitionInfo.replicas()[index];
      String rack = getRackHandleNull(replica);
      BrokerCapacityInfo brokerCapacity;
      try {
        // Do not allow capacity estimation for dead brokers.
        brokerCapacity = brokerCapacityConfigResolver.capacityForBroker(rack, replica.host(), replica.id(), BROKER_CAPACITY_FETCH_TIMEOUT_MS,
                                                                        aliveBrokers.contains(replica.id()) && allowCapacityEstimation);
      } catch (TimeoutException | BrokerCapacityResolutionException e) {
        // Capacity resolver may not be able to return the capacity information of dead brokers.
        if (!aliveBrokers.contains(replica.id())) {
          brokerCapacity = new BrokerCapacityInfo(EMPTY_BROKER_CAPACITY);
          deadBrokersWithUnknownCapacity.add(replica.id());
        } else {
          String errorMessage = String.format("Unable to retrieve capacity for broker %d. This may be caused by churn in "
                                              + "the cluster, please retry.", replica.id());
          LOG.warn(errorMessage, e);
          throw new TimeoutException(errorMessage);
        }
      }
      clusterModel.handleDeadBroker(rack, replica.id(), brokerCapacity);
      boolean isLeader;
      if (partitionInfo.leader() == null) {
        LOG.warn("Detected offline partition {}-{}, skipping", partitionInfo.topic(), partitionInfo.partition());
        continue;
      } else {
        isLeader = replica.id() == partitionInfo.leader().id();
      }
      boolean isOffline = Arrays.stream(partitionInfo.offlineReplicas())
                                .anyMatch(offlineReplica -> offlineReplica.id() == replica.id());

      String logdir = replicaPlacementInfo == null ? null : replicaPlacementInfo.get(tp).get(replica.id());
      // If the replica's logdir is null, it is either because replica placement information is not populated for the cluster
      // model or this replica is hosted on a dead disk and is not considered for intra-broker replica operations.
      clusterModel.createReplica(rack, replica.id(), tp, index, isLeader, isOffline, logdir, false);
      clusterModel.setReplicaLoad(rack,
                                  replica.id(),
                                  tp,
                                  getAggregatedMetricValues(valuesAndExtrapolations,
                                                            cluster.partition(tp),
                                                            isLeader,
                                                            needToAdjustCpuUsage),
                                  valuesAndExtrapolations.windows());
      needToAdjustCpuUsage = false;
    }
    if (!deadBrokersWithUnknownCapacity.isEmpty()) {
      LOG.info("Assign empty capacity to brokers {} because they are dead and capacity resolver is unable to fetch their capacity.",
               deadBrokersWithUnknownCapacity);
    }
  }
}
 

/**
 * Periodically checks the metadata to see if (1) partition or (2) leadership reassignment has finished or not.
 * @return Finished tasks.
 */
private List<ExecutionTask> waitForExecutionTaskToFinish() {
  List<ExecutionTask> finishedTasks = new ArrayList<>();
  Set<Long> forceStoppedTaskIds = new HashSet<>();
  Set<Long> deletedTaskIds = new HashSet<>();
  Set<Long> deadOrAbortingTaskIds = new HashSet<>();
  do {
    // If there is no finished tasks, we need to check if anything is blocked.
    maybeReexecuteTasks();
    try {
      Thread.sleep(executionProgressCheckIntervalMs());
    } catch (InterruptedException e) {
      // let it go
    }

    Cluster cluster = _metadataClient.refreshMetadata().cluster();
    Map<ExecutionTask, ReplicaLogDirInfo> logDirInfoByTask = getLogdirInfoForExecutionTask(
        _executionTaskManager.inExecutionTasks(Collections.singleton(INTRA_BROKER_REPLICA_ACTION)),
        _adminClient, _config);

    if (LOG.isDebugEnabled()) {
      LOG.debug("Tasks in execution: {}", inExecutionTasks());
    }
    List<ExecutionTask> deadOrAbortingInterBrokerReplicaActions = new ArrayList<>();
    List<ExecutionTask> slowTasksToReport = new ArrayList<>();
    boolean shouldReportSlowTasks = _time.milliseconds() - _lastSlowTaskReportingTimeMs > SLOW_TASK_ALERTING_BACKOFF_TIME_MS;
    for (ExecutionTask task : inExecutionTasks()) {
      TopicPartition tp = task.proposal().topicPartition();
      if (_stopSignal.get() == FORCE_STOP_EXECUTION) {
        LOG.debug("Task {} is marked as dead to force execution to stop.", task);
        finishedTasks.add(task);
        forceStoppedTaskIds.add(task.executionId());
        _executionTaskManager.markTaskDead(task);
      } else if (cluster.partition(tp) == null) {
        // Handle topic deletion during the execution.
        LOG.debug("Task {} is marked as finished because the topic has been deleted", task);
        finishedTasks.add(task);
        deletedTaskIds.add(task.executionId());
        _executionTaskManager.markTaskAborting(task);
        _executionTaskManager.markTaskDone(task);
      } else if (isTaskDone(cluster, logDirInfoByTask, tp, task)) {
        // Check to see if the task is done.
        finishedTasks.add(task);
        _executionTaskManager.markTaskDone(task);
      } else {
        if (shouldReportSlowTasks) {
          task.maybeReportExecutionTooSlow(_time.milliseconds(), slowTasksToReport);
        }
        if (maybeMarkTaskAsDeadOrAborting(cluster, logDirInfoByTask, task)) {
          deadOrAbortingTaskIds.add(task.executionId());
          // Only add the dead or aborted tasks to execute if it is an inter-broker replica action.
          if (task.type() == INTER_BROKER_REPLICA_ACTION) {
            deadOrAbortingInterBrokerReplicaActions.add(task);
          }
          // A dead or aborted task is considered as finished.
          if (task.state() == DEAD || task.state() == ABORTED) {
            finishedTasks.add(task);
          }
        }
      }
    }
    sendSlowExecutionAlert(slowTasksToReport);
    handleDeadOrAbortingReplicaActions(deadOrAbortingInterBrokerReplicaActions);
    updateOngoingExecutionState();
  } while (!inExecutionTasks().isEmpty() && finishedTasks.isEmpty());

  LOG.info("Finished tasks: {}.{}{}{}", finishedTasks,
           forceStoppedTaskIds.isEmpty() ? "" : String.format(". [Force-stopped: %s]", forceStoppedTaskIds),
           deletedTaskIds.isEmpty() ? "" : String.format(". [Deleted: %s]", deletedTaskIds),
           deadOrAbortingTaskIds.isEmpty() ? "" : String.format(". [Dead/aborting: %s]", deadOrAbortingTaskIds));

  return finishedTasks;
}
 

/**
 * Check if the partition is currently under replicated.
 * @param cluster The current cluster state.
 * @param tp The topic partition to check.
 * @return True if the partition is currently under replicated.
 */
public static boolean isPartitionUnderReplicated(Cluster cluster, TopicPartition tp) {
  PartitionInfo partitionInfo = cluster.partition(tp);
  return partitionInfo.inSyncReplicas().length != partitionInfo.replicas().length;
}