Java Code Examples for org.apache.kafka.common.PartitionInfo#inSyncReplicas()

/**
 * Gather the Kafka partition state within the given under replicated, offline, under minIsr,
 * and other partitions (if verbose).
 *
 * @param underReplicatedPartitions state of under replicated partitions.
 * @param offlinePartitions state of offline partitions.
 * @param otherPartitions state of partitions other than offline or urp.
 * @param partitionsWithOfflineReplicas state of partitions with offline replicas.
 * @param underMinIsrPartitions state of under min isr partitions.
 * @param verbose true if requested to gather state of partitions other than offline or urp.
 * @param topicPattern regex of topic to filter partition states by, is null if no filter is to be applied
 */
protected void populateKafkaPartitionState(Set<PartitionInfo> underReplicatedPartitions,
    Set<PartitionInfo> offlinePartitions,
    Set<PartitionInfo> otherPartitions,
    Set<PartitionInfo> partitionsWithOfflineReplicas,
    Set<PartitionInfo> underMinIsrPartitions,
    boolean verbose,
    Pattern topicPattern) {
  for (String topic : _kafkaCluster.topics()) {
    if (topicPattern == null || topicPattern.matcher(topic).matches()) {
      int minInsyncReplicas = minInsyncReplicas(topic);
      for (PartitionInfo partitionInfo : _kafkaCluster.partitionsForTopic(topic)) {
        int numInsyncReplicas = partitionInfo.inSyncReplicas().length;
        boolean isURP = numInsyncReplicas != partitionInfo.replicas().length;
        if (numInsyncReplicas < minInsyncReplicas) {
          underMinIsrPartitions.add(partitionInfo);
        }
        if (isURP || verbose) {
          boolean hasOfflineReplica = partitionInfo.offlineReplicas().length != 0;
          if (hasOfflineReplica) {
            partitionsWithOfflineReplicas.add(partitionInfo);
          }
          boolean isOffline = partitionInfo.inSyncReplicas().length == 0;
          if (isOffline) {
            offlinePartitions.add(partitionInfo);
          } else if (isURP) {
            underReplicatedPartitions.add(partitionInfo);
          } else {
            // verbose -- other
            otherPartitions.add(partitionInfo);
          }
        }
      }
    }
  }
}
 

/**
 * Get the under replicated nodes from PartitionInfo
 */
public static Set<Node> getNotInSyncBrokers(PartitionInfo partitionInfo) {
  if (partitionInfo.inSyncReplicas().length == partitionInfo.replicas().length) {
    return new HashSet<>();
  }
  Set<Node> nodes = new HashSet<>(Arrays.asList(partitionInfo.replicas()));
  for (Node node : partitionInfo.inSyncReplicas()) {
    nodes.remove(node);
  }
  return nodes;
}
 

public Set<Integer>  getInSyncReplicas(PartitionInfo  partitionInfo) {
  Set<Integer> result = new HashSet<>();
  for (Node node: partitionInfo.inSyncReplicas()) {
    result.add(node.id());
  }
  return result;
}
 

/**
 * Get the under replicated nodes from PartitionInfo
 */
public static Set<Integer> getOutOfSyncReplicas(PartitionInfo partitionInfo) {
  if (partitionInfo.inSyncReplicas().length == partitionInfo.replicas().length) {
    return new HashSet<>();
  }
  Set<Node> nodes = new HashSet<>(Arrays.asList(partitionInfo.replicas()));
  for (Node node : partitionInfo.inSyncReplicas()) {
    nodes.remove(node);
  }
  return nodes.stream().map(nd -> nd.id()).collect(Collectors.toSet());
}
 

/**
 * Call the kafka api to get the list of under-replicated partitions.
 * When a topic partition loses all of its replicas, it will not have a leader broker.
 * We need to handle this special case in detecting under replicated topic partitions.
 */
public static List<PartitionInfo> getUnderReplicatedPartitions(
    String zkUrl, SecurityProtocol securityProtocol, Map<String, String> consumerConfigs,
    List<String> topics,
    scala.collection.mutable.Map<String, scala.collection.Map<Object, Seq<Object>>> partitionAssignments,
    Map<String, Integer> replicationFactors,
    Map<String, Integer> partitionCounts) {
  List<PartitionInfo> underReplicated = new ArrayList();
  KafkaConsumer kafkaConsumer = KafkaUtils.getKafkaConsumer(zkUrl, securityProtocol, consumerConfigs);
  for (String topic : topics) {
    List<PartitionInfo> partitionInfoList = kafkaConsumer.partitionsFor(topic);
    if (partitionInfoList == null) {
      LOG.error("Failed to get partition info for {}", topic);
      continue;
    }
    int numPartitions = partitionCounts.get(topic);

    // when a partition loses all replicas and does not have a live leader,
    // kafkaconsumer.partitionsFor(...) will not return info for that partition.
    // the noLeaderFlag array is used to detect partitions that have no leaders
    boolean[] noLeaderFlags = new boolean[numPartitions];
    for (int i = 0; i < numPartitions; i++) {
      noLeaderFlags[i] = true;
    }
    for (PartitionInfo info : partitionInfoList) {
      if (info.inSyncReplicas().length < info.replicas().length &&
          replicationFactors.get(info.topic()) > info.inSyncReplicas().length) {
        underReplicated.add(info);
      }
      noLeaderFlags[info.partition()] = false;
    }

    // deal with the partitions that do not have leaders
    for (int partitionId = 0; partitionId < numPartitions; partitionId++) {
      if (noLeaderFlags[partitionId]) {
        Seq<Object> seq = partitionAssignments.get(topic).get().get(partitionId).get();
        Node[] nodes = JavaConverters.seqAsJavaList(seq).stream()
            .map(val -> new Node((Integer) val, "", -1)).toArray(Node[]::new);
        PartitionInfo partitionInfo =
            new PartitionInfo(topic, partitionId, null, nodes, new Node[0]);
        underReplicated.add(partitionInfo);
      }
    }
  }
  return underReplicated;
}
 

/**
 * Check if the partition is currently under replicated.
 * @param cluster The current cluster state.
 * @param tp The topic partition to check.
 * @return True if the partition is currently under replicated.
 */
public static boolean isPartitionUnderReplicated(Cluster cluster, TopicPartition tp) {
  PartitionInfo partitionInfo = cluster.partition(tp);
  return partitionInfo.inSyncReplicas().length != partitionInfo.replicas().length;
}
 

/**
 * Check whether the successful completion of inter-broker replica movement from this proposal is reflected in the current
 * ordered replicas in the given cluster and all replicas are in-sync.
 *
 * @param partitionInfo Current partition state.
 * @return True if successfully completed, false otherwise.
 */
public boolean isInterBrokerMovementCompleted(PartitionInfo partitionInfo) {
  return brokerOrderMatched(partitionInfo.replicas(), _newReplicas)
         && partitionInfo.replicas().length == partitionInfo.inSyncReplicas().length;
}