Java Code Examples for kafka.utils.ZkUtils#getAllTopics()

public static void main(String[] args) throws Exception {
  CommandLine commandLine = parseCommandLine(args);
  String zookeeper = commandLine.getOptionValue(ZOOKEEPER);

  ZkUtils zkUtils = KafkaUtils.getZkUtils(zookeeper);
  Seq<String> topicsSeq = zkUtils.getAllTopics();
  List<String> topics = scala.collection.JavaConverters.seqAsJavaList(topicsSeq);

  scala.collection.mutable.Map<String, scala.collection.Map<Object, Seq<Object>>>
      partitionAssignments = zkUtils.getPartitionAssignmentForTopics(topicsSeq);

  Map<String, Integer> replicationFactors = new HashMap<>();
  Map<String, Integer> partitionCounts = new HashMap<>();

  topics.stream().forEach(topic -> {
    int partitionCount = partitionAssignments.get(topic).get().size();
    int factor = partitionAssignments.get(topic).get().head()._2().size();
    partitionCounts.put(topic, partitionCount);
    replicationFactors.put(topic, factor);
  });

  List<PartitionInfo> urps = KafkaClusterManager.getUnderReplicatedPartitions(
      zookeeper, SecurityProtocol.PLAINTEXT, null, topics, partitionAssignments, replicationFactors, partitionCounts);

  for (PartitionInfo partitionInfo : urps) {
    LOG.info("under-replicated : {}", partitionInfo);
  }
}
 

private static void printCurrentAssignment(ZkUtils zkUtils, List<String> specifiedTopics) {
    Seq<String> topics = specifiedTopics != null ?
            JavaConversions.iterableAsScalaIterable(specifiedTopics).toSeq() :
            zkUtils.getAllTopics();
    System.out.println("CURRENT ASSIGNMENT:");
    System.out.println(
            zkUtils.formatAsReassignmentJson(zkUtils.getReplicaAssignmentForTopics(
                    topics)));
}
 

/**
 *  KafkaClusterManager periodically check the health of the cluster. If it finds
 *  an under-replicated partitions, it will perform partition reassignment. It will also
 *  do partition reassignment for workload balancing.
 *
 *  If partitions are under-replicated in the middle of work-load balancing due to
 *  broker failure, it will send out an alert. Human intervention is needed in this case.
 */
@Override
public void run() {
  long checkIntervalInMs = clusterConfig.getCheckIntervalInSeconds() * 1000L;
  stopped = false;
  boolean foundUrps = false;
  long firstSeenUrpsTimestamp = 0L;

  while (!stopped) {
    try {
      Thread.sleep(checkIntervalInMs);
      if (maintenanceMode.get()) {
        LOG.debug("Cluster:" + clusterConfig.getClusterName() + " is in maintenace mode");
        continue;
      }
      ZkUtils zkUtils = KafkaUtils.getZkUtils(zkUrl);

      // check if there is any broker that do not have stats.
      List<Broker> noStatsBrokers = getNoStatsBrokers();
      if (!noStatsBrokers.isEmpty()) {
        Email.alertOnNoStatsBrokers(
            drkafkaConfig.getAlertEmails(), clusterConfig.getClusterName(), noStatsBrokers);
        continue;
      }

      Seq<String> topicsSeq = zkUtils.getAllTopics();
      List<String> topics = scala.collection.JavaConverters.seqAsJavaList(topicsSeq);
      scala.collection.mutable.Map<String, scala.collection.Map<Object, Seq<Object>>>
          partitionAssignments = zkUtils.getPartitionAssignmentForTopics(topicsSeq);

      Map<String, Integer> replicationFactors = new HashMap<>();
      Map<String, Integer> partitionCounts = new HashMap<>();
      topics.stream().forEach(topic -> {
        int partitionCount = partitionAssignments.get(topic).get().size();
        int factor = partitionAssignments.get(topic).get().head()._2().size();
        partitionCounts.put(topic, partitionCount);
        replicationFactors.put(topic, factor);
      });

      underReplicatedPartitions = getUnderReplicatedPartitions(zkUrl, securityProtocol, consumerConfigs,
          topics, partitionAssignments, replicationFactors, partitionCounts);
      LOG.info("Under-replicated partitions: {}", underReplicatedPartitions.size());

      for (PartitionInfo partitionInfo : underReplicatedPartitions) {
        LOG.info("under-replicated : {}", partitionInfo);
      }

      kafkaCluster.clearResourceAllocationCounters();
      if (underReplicatedPartitions.size() > 0) {
        // handle under-replicated partitions
        if (!foundUrps) {
          foundUrps = true;
          firstSeenUrpsTimestamp = System.currentTimeMillis();
        } else {
          // send out an alert if the cluster has been under-replicated for a while
          long underReplicatedTimeMills = System.currentTimeMillis() - firstSeenUrpsTimestamp;
          if (underReplicatedTimeMills > clusterConfig.getUnderReplicatedAlertTimeInMs()) {

            Email.alertOnProlongedUnderReplicatedPartitions(drkafkaConfig.getAlertEmails(),
                clusterConfig.getClusterName(),
                clusterConfig.getUnderReplicatedAlertTimeInSeconds(),
                underReplicatedPartitions);
          }
        }
        LOG.info("Under-replicated partitions in cluster {} : {}",
            clusterConfig.getClusterName(), underReplicatedPartitions.size());

        handleUnderReplicatedPartitions(underReplicatedPartitions, replicationFactors);
      } else {
        foundUrps = false;
        firstSeenUrpsTimestamp = Long.MAX_VALUE;
        if (clusterConfig.enabledWorloadBalancing()) {
          preferredLeaders.clear();
          reassignmentMap.clear();
          balanceWorkload();
        }
      }
      if (clusterConfig.enabledDeadbrokerReplacement()) {
        // replace the brokers that do not have kafkastats update for a while
        checkAndReplaceDeadBrokers();
      }
    } catch (Exception e) {
      LOG.error("Unexpected failure in cluster manager for {}: ", zkUrl, e);
    }
  }
}
 

private static void printLeastDisruptiveReassignment(
        ZkUtils zkUtils, List<String> specifiedTopics, Set<Integer> specifiedBrokers,
        Set<Integer> excludedBrokers, Map<Integer, String> rackAssignment, int desiredReplicationFactor)
        throws JSONException {
    // We need three inputs for rebalacing: the brokers, the topics, and the current assignment
    // of topics to brokers.
    Set<Integer> brokerSet = specifiedBrokers;
    if (brokerSet == null || brokerSet.isEmpty()) {
        brokerSet = Sets.newHashSet(Lists.transform(
                JavaConversions.seqAsJavaList(zkUtils.getAllBrokersInCluster()),
                new Function<Broker, Integer>() {
                    @Override
                    public Integer apply(Broker broker) {
                        return broker.id();
                    }
                }));
    }

    // Exclude brokers that we want to decommission
    Set<Integer> brokers = Sets.difference(brokerSet, excludedBrokers);
    rackAssignment.keySet().retainAll(brokers);

    // The most common use case is to rebalance all topics, but explicit topic addition is also
    // supported.
    Seq<String> topics = specifiedTopics != null ?
            JavaConversions.collectionAsScalaIterable(specifiedTopics).toSeq() :
            zkUtils.getAllTopics();

    // Print the current assignment in case a rollback is needed
    printCurrentAssignment(zkUtils, JavaConversions.seqAsJavaList(topics));

    Map<String, Map<Integer, List<Integer>>> initialAssignments =
            KafkaTopicAssigner.topicMapToJavaMap(zkUtils.getPartitionAssignmentForTopics(
                    topics));

    // Assign topics one at a time. This is slightly suboptimal from a packing standpoint, but
    // it's close enough to work in practice. We can also always follow it up with a Kafka
    // leader election rebalance if necessary.
    JSONObject json = new JSONObject();
    json.put("version", KAFKA_FORMAT_VERSION);
    JSONArray partitionsJson = new JSONArray();
    KafkaTopicAssigner assigner = new KafkaTopicAssigner();
    for (String topic : JavaConversions.seqAsJavaList(topics)) {
        Map<Integer, List<Integer>> partitionAssignment = initialAssignments.get(topic);
        Map<Integer, List<Integer>> finalAssignment = assigner.generateAssignment(
                topic, partitionAssignment, brokers, rackAssignment, desiredReplicationFactor);
        for (Map.Entry<Integer, List<Integer>> e : finalAssignment.entrySet()) {
            JSONObject partitionJson = new JSONObject();
            partitionJson.put("topic", topic);
            partitionJson.put("partition", e.getKey());
            partitionJson.put("replicas", new JSONArray(e.getValue()));
            partitionsJson.put(partitionJson);
        }
    }
    json.put("partitions", partitionsJson);
    System.out.println("NEW ASSIGNMENT:\n" + json.toString());
}