org.apache.kafka.common.requests.DescribeLogDirsResponse Java Examples

/**
 * iterates through the broker's log directories and checks for the ongoing topic partitions and replica's existence.
 * @param logDirectoriesResponseMap map of log directories response in the broker
 * @param broker broker to process the log dirs in
 * @param topic ongoing kmf manipulation topic
 */
int processBroker(Map<Integer, Map<String, DescribeLogDirsResponse.LogDirInfo>> logDirectoriesResponseMap,
    Node broker, String topic) {
  int totalPartitionsInBroker = 0;
  LOGGER.trace("logDirectoriesResponseMap: {}", logDirectoriesResponseMap);
  Map<String, DescribeLogDirsResponse.LogDirInfo> logDirInfoMap = logDirectoriesResponseMap.get(broker.id());
  String logDirectoriesKey = logDirInfoMap.keySet().iterator().next();
  LOGGER.trace("logDirInfoMap: {}", logDirInfoMap.get(logDirectoriesKey));
  DescribeLogDirsResponse.LogDirInfo logDirInfo = logDirInfoMap.get(logDirectoriesKey);

  if (logDirInfo != null && !logDirectoriesResponseMap.isEmpty()) {
    Map<TopicPartition, DescribeLogDirsResponse.ReplicaInfo> topicPartitionReplicaInfoMap = logDirInfo.replicaInfos;
    totalPartitionsInBroker += this.processLogDirsWithinBroker(topicPartitionReplicaInfoMap, topic, broker);
  }

  return totalPartitionsInBroker;
}
 

private int processLogDirsWithinBroker(
    Map<TopicPartition, DescribeLogDirsResponse.ReplicaInfo> topicPartitionReplicaInfoMap, String topic,
    Node broker) {
  int totalPartitionsInBroker = 0;
  for (Map.Entry<TopicPartition, DescribeLogDirsResponse.ReplicaInfo> topicPartitionReplicaInfoEntry : topicPartitionReplicaInfoMap
      .entrySet()) {

    TopicPartition topicPartition = topicPartitionReplicaInfoEntry.getKey();
    DescribeLogDirsResponse.ReplicaInfo replicaInfo = topicPartitionReplicaInfoEntry.getValue();

    if (topicPartition.topic().equals(topic)) {
      totalPartitionsInBroker++;
      LOGGER.trace("totalPartitions In The Broker = {}", totalPartitionsInBroker);
    }

    LOGGER.trace("broker information: {}", broker);
    LOGGER.trace("logDirInfo for kafka-logs: topicPartition = {}, replicaInfo = {}", topicPartition, replicaInfo);
  }

  return totalPartitionsInBroker;
}
 

protected void populateKafkaBrokerLogDirState(Map<Integer, Set<String>> onlineLogDirsByBrokerId,
                                              Map<Integer, Set<String>> offlineLogDirsByBrokerId,
                                              Set<Integer> brokers)
    throws ExecutionException, InterruptedException {
  // If the broker does not show up in latest metadata, the broker is dead.
  Set<Integer> aliveBrokers = new HashSet<>(brokers.size());
  _kafkaCluster.nodes().forEach(node -> aliveBrokers.add(node.id()));
  for (Integer broker: brokers) {
    if (!aliveBrokers.contains(broker)) {
      onlineLogDirsByBrokerId.put(broker, Collections.singleton("broker_dead"));
      offlineLogDirsByBrokerId.put(broker, Collections.singleton("broker_dead"));
    }
  }

  Map<Integer, KafkaFuture<Map<String, DescribeLogDirsResponse.LogDirInfo>>> logDirsByBrokerId = describeLogDirs(aliveBrokers,
                                                                                                                 _adminClientConfigs).values();
  for (Map.Entry<Integer, KafkaFuture<Map<String, DescribeLogDirsResponse.LogDirInfo>>> entry : logDirsByBrokerId.entrySet()) {
    onlineLogDirsByBrokerId.put(entry.getKey(), new HashSet<>());
    offlineLogDirsByBrokerId.put(entry.getKey(), new HashSet<>());

    try {
      entry.getValue().get(_config.getLong(LOGDIR_RESPONSE_TIMEOUT_MS_CONFIG), TimeUnit.MILLISECONDS).forEach((key, value) -> {
        if (value.error == Errors.NONE) {
          onlineLogDirsByBrokerId.get(entry.getKey()).add(key);
        } else {
          offlineLogDirsByBrokerId.get(entry.getKey()).add(key);
        }
      });
    } catch (TimeoutException te) {
      LOG.error("Getting log dir information for broker {} timed out.", entry.getKey());
      onlineLogDirsByBrokerId.get(entry.getKey()).add("timed_out");
      offlineLogDirsByBrokerId.get(entry.getKey()).add("timed_out");
    }
  }
}
 

private DescribeLogDirsResult getDescribeLogDirsResult() {
  try {
    // Reflectively set DescribeLogDirsResult's constructor from package private to public.
    Constructor<DescribeLogDirsResult> constructor = DescribeLogDirsResult.class.getDeclaredConstructor(Map.class);
    constructor.setAccessible(true);

    Map<Integer, KafkaFuture<Map<String, DescribeLogDirsResponse.LogDirInfo>>> futureByBroker = new HashMap<>();
    Map<String, DescribeLogDirsResponse.LogDirInfo> logdirInfoBylogdir =  new HashMap<>();
    Map<TopicPartition, DescribeLogDirsResponse.ReplicaInfo> replicaInfoByPartition = new HashMap<>();
    replicaInfoByPartition.put(T0P0, new DescribeLogDirsResponse.ReplicaInfo(0, 0, false));
    replicaInfoByPartition.put(T0P1, new DescribeLogDirsResponse.ReplicaInfo(0, 0, false));
    replicaInfoByPartition.put(T1P0, new DescribeLogDirsResponse.ReplicaInfo(0, 0, false));
    replicaInfoByPartition.put(T1P1, new DescribeLogDirsResponse.ReplicaInfo(0, 0, false));
    logdirInfoBylogdir.put("/tmp/kafka-logs", new DescribeLogDirsResponse.LogDirInfo(Errors.NONE, replicaInfoByPartition));
    futureByBroker.put(0, completedFuture(logdirInfoBylogdir));

    logdirInfoBylogdir =  new HashMap<>();
    replicaInfoByPartition = new HashMap<>();
    replicaInfoByPartition.put(T0P0, new DescribeLogDirsResponse.ReplicaInfo(0, 0, false));
    replicaInfoByPartition.put(T0P1, new DescribeLogDirsResponse.ReplicaInfo(0, 0, false));
    replicaInfoByPartition.put(T1P0, new DescribeLogDirsResponse.ReplicaInfo(0, 0, false));
    logdirInfoBylogdir.put("/tmp/kafka-logs-1", new DescribeLogDirsResponse.LogDirInfo(Errors.NONE, replicaInfoByPartition));
    logdirInfoBylogdir.put("/tmp/kafka-logs-2",
                           new DescribeLogDirsResponse.LogDirInfo(Errors.NONE,
                                                                  Collections.singletonMap(T1P1,
                                                                                           new DescribeLogDirsResponse.ReplicaInfo(0, 0, false))));
    futureByBroker.put(1, completedFuture(logdirInfoBylogdir));
    return constructor.newInstance(futureByBroker);
  } catch (InstantiationException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) {
    // Let it go.
  }
  return null;
}
 

/**
 * for all brokers, checks if the topic exists in the cluster by iterating through the log dirs of individual brokers.
 * @param topic current ongoing topic
 * @param brokers brokers to check log dirs from
 * @param adminClient Admin Client
 * @return true if the cluster contains the topic.
 * @throws ExecutionException when attempting to retrieve the result of a task
 * that aborted by throwing an exception.
 * @throws InterruptedException when a thread is waiting, sleeping, or occupied,
 * and the thread is interrupted, either before or during the activity.
 */
private boolean doesClusterContainTopic(String topic, Collection<Node> brokers, AdminClient adminClient,
    int expectedTotalPartitionsInCluster) throws ExecutionException, InterruptedException {
  int totalPartitionsInCluster = 0;
  for (Node broker : brokers) {
    LOGGER.trace("broker log directories: {}",
        adminClient.describeLogDirs(Collections.singleton(broker.id())).all().get());
    Map<Integer, Map<String, DescribeLogDirsResponse.LogDirInfo>> logDirectoriesResponseMap =
        adminClient.describeLogDirs(Collections.singleton(broker.id())).all().get();

    totalPartitionsInCluster += this.processBroker(logDirectoriesResponseMap, broker, topic);
  }

  if (totalPartitionsInCluster != expectedTotalPartitionsInCluster) {
    LOGGER.debug("totalPartitionsInCluster {} does not equal expectedTotalPartitionsInCluster {}",
        totalPartitionsInCluster, expectedTotalPartitionsInCluster);
    return false;
  }

  boolean isDescribeSuccessful = true;
  try {
    Map<String, TopicDescription> topicDescriptions =
        ClusterTopicManipulationService.describeTopics(adminClient, Collections.singleton(topic));
    LOGGER.trace("topicDescriptionMap = {}", topicDescriptions);
  } catch (InterruptedException | ExecutionException e) {
    isDescribeSuccessful = false;
    LOGGER.error("Exception occurred within describeTopicsFinished method for topics {}",
        Collections.singleton(topic), e);
  }

  LOGGER.trace("isDescribeSuccessful: {}", isDescribeSuccessful);
  return isDescribeSuccessful;
}
 

public JSONObject topicKafkaCapacity(String clusterAlias, String topic) {
	if (Kafka.CONSUMER_OFFSET_TOPIC.equals(topic)) {
		return new JSONObject();
	}
	Properties prop = new Properties();
	prop.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, parseBrokerServer(clusterAlias));

	if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) {
		kafkaService.sasl(prop, clusterAlias);
	}
	if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) {
		kafkaService.ssl(prop, clusterAlias);
	}
	long sum = 0L;
	AdminClient adminClient = null;
	try {
		adminClient = AdminClient.create(prop);
		List<MetadataInfo> leaders = kafkaService.findKafkaLeader(clusterAlias, topic);
		Set<Integer> ids = new HashSet<>();
		for (MetadataInfo metadata : leaders) {
			ids.add(metadata.getLeader());
		}
		DescribeLogDirsResult logSizeBytes = adminClient.describeLogDirs(ids);
		Map<Integer, Map<String, DescribeLogDirsResponse.LogDirInfo>> tmp = logSizeBytes.all().get();
		if (tmp == null) {
			return new JSONObject();
		}

		for (Map.Entry<Integer, Map<String, DescribeLogDirsResponse.LogDirInfo>> entry : tmp.entrySet()) {
			Map<String, DescribeLogDirsResponse.LogDirInfo> logDirInfos = entry.getValue();
			for (Map.Entry<String, DescribeLogDirsResponse.LogDirInfo> logDirInfo : logDirInfos.entrySet()) {
				DescribeLogDirsResponse.LogDirInfo info = logDirInfo.getValue();
				Map<TopicPartition, DescribeLogDirsResponse.ReplicaInfo> replicaInfoMap = info.replicaInfos;
				for (Map.Entry<TopicPartition, DescribeLogDirsResponse.ReplicaInfo> replicas : replicaInfoMap.entrySet()) {
					if (topic.equals(replicas.getKey().topic())) {
						sum += replicas.getValue().size;
					}
				}
			}
		}

	} catch (Exception e) {
		LOG.error("Get topic capacity has error, msg is " + e.getCause().getMessage());
		e.printStackTrace();
	} finally {
		adminClient.close();
	}
	return StrUtils.stringifyByObject(sum);
}
 

@Test(invocationCount = 2)
void serviceStartTest() throws JsonProcessingException {
  ClusterTopicManipulationService clusterTopicManipulationService =
      Mockito.mock(ClusterTopicManipulationService.class);

  Mockito.doCallRealMethod()
      .when(clusterTopicManipulationService)
      .processBroker(Mockito.anyMap(), Mockito.any(), Mockito.anyString());

  Mockito.doCallRealMethod()
      .when(clusterTopicManipulationService)
      .setExpectedPartitionsCount(Mockito.anyInt());

  Mockito.doCallRealMethod()
      .when(clusterTopicManipulationService)
      .expectedPartitionsCount();

  List<Node> brokers = new ArrayList<>();
  for (int id = 1; id < 3; id++) {
    brokers.add(new Node(id, "kafka-broker-host", 8000));
  }

  Map<Integer, Map<String, DescribeLogDirsResponse.LogDirInfo>> logDirectoriesResponseMap1 = new HashMap<>();
  Map<Integer, Map<String, DescribeLogDirsResponse.LogDirInfo>> logDirectoriesResponseMap2 = new HashMap<>();

  Map<Node, Map<Integer, Map<String, DescribeLogDirsResponse.LogDirInfo>>> brokerMapHashMap = new HashMap<>();
  brokerMapHashMap.putIfAbsent(brokers.get(0), logDirectoriesResponseMap1);
  brokerMapHashMap.putIfAbsent(brokers.get(1), logDirectoriesResponseMap2);

  Map<String, DescribeLogDirsResponse.LogDirInfo> logDirInfoMap1 = new HashMap<>();
  Map<String, DescribeLogDirsResponse.LogDirInfo> logDirInfoMap2 = new HashMap<>();

  logDirectoriesResponseMap1.put(brokers.get(0).id(), logDirInfoMap1);
  logDirectoriesResponseMap2.put(brokers.get(1).id(), logDirInfoMap2);

  Map<TopicPartition, DescribeLogDirsResponse.ReplicaInfo> replicaInfos1 = new HashMap<>();
  Map<TopicPartition, DescribeLogDirsResponse.ReplicaInfo> replicaInfos2 = new HashMap<>();

  for (int topicPartition = 0; topicPartition < 3; topicPartition++) {
    replicaInfos1.put(new TopicPartition(SERVICE_TEST_TOPIC, topicPartition),
        new DescribeLogDirsResponse.ReplicaInfo(235, 0, false));

    replicaInfos2.put(new TopicPartition(SERVICE_TEST_TOPIC, topicPartition),
        new DescribeLogDirsResponse.ReplicaInfo(235, 0, false));
  }

  int totalPartitions = brokers.size() * replicaInfos1.size();
  System.out.println(totalPartitions);
  clusterTopicManipulationService.setExpectedPartitionsCount(totalPartitions);
  System.out.println(clusterTopicManipulationService.expectedPartitionsCount());

  logDirInfoMap1.put(XinfraMonitorConstants.KAFKA_LOG_DIRECTORY + "-1",
      new DescribeLogDirsResponse.LogDirInfo(null, replicaInfos1));
  logDirInfoMap2.put(XinfraMonitorConstants.KAFKA_LOG_DIRECTORY + "-2",
      new DescribeLogDirsResponse.LogDirInfo(null, replicaInfos2));

  ObjectMapper objectMapper = new ObjectMapper();
  ObjectWriter objectWriter = objectMapper.writerWithDefaultPrettyPrinter();

  for (Map.Entry<Node, Map<Integer, Map<String, DescribeLogDirsResponse.LogDirInfo>>> nodeMapEntry : brokerMapHashMap.entrySet()) {
    System.out.println(objectWriter.writeValueAsString(nodeMapEntry.getValue()));
  }

  for (Node broker : brokers) {
    clusterTopicManipulationService.processBroker(brokerMapHashMap.get(broker), broker, SERVICE_TEST_TOPIC);
  }

  Assert.assertEquals(totalPartitions, clusterTopicManipulationService.expectedPartitionsCount());
  System.out.println();
}