org.apache.kafka.clients.admin.ListTopicsOptions Java Examples

public Map<String, TopicDescription> getTopicAndDescriptions() throws Exception {

		try {
			// 查询topic
			ListTopicsOptions lto = new ListTopicsOptions();
			lto.timeoutMs(10 * 1000);
			ListTopicsResult ltr = adminClient.listTopics(lto);
			
			// 查询topic配置信息
			DescribeTopicsOptions dto = new DescribeTopicsOptions();
			dto.timeoutMs(15 * 1000);
			DescribeTopicsResult dtr = adminClient.describeTopics(ltr.names().get(), dto);
			return dtr.all().get();
			
		} catch (Exception e) {
			throw e;
		}
	}
 

@Test
public void testListExisting() {
  Cluster cluster = createCluster(1);
  TopicPartitionInfo tp = new TopicPartitionInfo(0, cluster.nodeById(0), cluster.nodes(), Collections.emptyList());
  ConfigEntry configEntry = new ConfigEntry("k", "v");
  KafkaFuture<Config> kfc = KafkaFuture.completedFuture(new Config(Collections.singletonList(configEntry)));
  Set<String> topicNames = new HashSet<>(Arrays.asList("a", "b", "_c"));
  Map<String, TopicDescription> tds = new HashMap<String, TopicDescription>() {
    {
      put("a", new TopicDescription("a", false, Collections.singletonList(tp)));
      put("b", new TopicDescription("b", false, Collections.singletonList(tp)));
      put("c", new TopicDescription("_c", false, Collections.singletonList(tp)));
    }
  };
  Map<ConfigResource, KafkaFuture<Config>> configs = new HashMap<ConfigResource, KafkaFuture<Config>>() {
    {
      put(new ConfigResource(TOPIC, "a"), kfc);
      put(new ConfigResource(TOPIC, "b"), kfc);
      put(new ConfigResource(TOPIC, "_c"), kfc);
    }
  };

  TopicService service = new TopicServiceImpl(adminClient, true);
  ListTopicsResult listTopicsResult = mock(ListTopicsResult.class);
  DescribeTopicsResult describeTopicsResult = mock(DescribeTopicsResult.class);
  DescribeConfigsResult describeConfigsResult = mock(DescribeConfigsResult.class);

  when(describeTopicsResult.all()).thenReturn(KafkaFuture.completedFuture(tds));
  when(listTopicsResult.names()).thenReturn(KafkaFuture.completedFuture(topicNames));
  when(describeConfigsResult.values()).thenReturn(configs);
  when(adminClient.listTopics(any(ListTopicsOptions.class))).thenReturn(listTopicsResult);
  when(adminClient.describeTopics(topicNames)).thenReturn(describeTopicsResult);
  when(adminClient.describeConfigs(any(Collection.class))).thenReturn(describeConfigsResult);

  Map<String, ConfiguredTopic> actual = service.listExisting(true);
  Assert.assertEquals(2, actual.size());
  Assert.assertEquals(new HashSet<>(Arrays.asList("a", "b")), actual.keySet());
}
 

public Set<ClusterTopicInfo> describeTopics() throws InterruptedException, ExecutionException, TimeoutException {

        Set<ClusterTopicInfo> result = new HashSet<>();
        final ListTopicsResult listTopicsResult = kafkaClientsAdminClient.listTopics(new ListTopicsOptions().listInternal(false));
        final Collection<TopicListing> listings = listTopicsResult.listings().get(ApplicationConstants.FUTURE_GET_TIMEOUT_MS, TimeUnit.MILLISECONDS);
        Logger.debug(String.format("describeTopics.listings %s", listings));


        final Set<String> topicNames = listTopicsResult.names().get(ApplicationConstants.FUTURE_GET_TIMEOUT_MS, TimeUnit.MILLISECONDS);
        final DescribeTopicsResult describeTopicsResult = kafkaClientsAdminClient.describeTopics(topicNames);
        final Map<String, TopicDescription> stringTopicDescriptionMap = describeTopicsResult.all().get(ApplicationConstants.FUTURE_GET_TIMEOUT_MS,
                                                                                                       TimeUnit.MILLISECONDS);

        for (Map.Entry<String, TopicDescription> entry : stringTopicDescriptionMap.entrySet()) {
            final TopicDescription topicDescription = entry.getValue();
            final ClusterTopicInfo clusterTopicInfo = new ClusterTopicInfo(topicDescription.name(),
                                                                           topicDescription.partitions(),
                                                                           getConfigEntriesForTopic(topicDescription.name()));
            result.add(clusterTopicInfo);
        }
        return result;

    }
 

private synchronized Set<LeaderTopicPartition> retrieveLeaderTopicPartitions(int requestTimeoutMs)
    throws InterruptedException, ExecutionException, TimeoutException {
  long startWait = System.currentTimeMillis();

  ListTopicsOptions listTopicsOptions = new ListTopicsOptions().listInternal(false)
      .timeoutMs((int) (requestTimeoutMs - (System.currentTimeMillis() - startWait)));
  Set<String> retrievedTopicSet = partitionMonitorClient.listTopics(listTopicsOptions).names()
      .get(requestTimeoutMs - (System.currentTimeMillis() - startWait), TimeUnit.MILLISECONDS);
  logger.debug("Server topic list: {}", retrievedTopicSet);
  Set<String> matchedTopicSet = retrievedTopicSet.stream().filter(topic -> matchedTopicFilter(topic))
      .collect(Collectors.toSet());
  if (matchedTopicSet.size() > 0) {
    logger.debug("Matched topic list: {}", matchedTopicSet);
  } else {
    logger.warn("Provided pattern {} does currently not match any topic." +
                 " Thus connector won't spawn any task until partition monitor recognizes matching topic.", this.topicWhitelistPattern.toString());

  }

  DescribeTopicsOptions describeTopicsOptions = new DescribeTopicsOptions()
      .timeoutMs((int) (requestTimeoutMs - (System.currentTimeMillis() - startWait)));
  Map<String, TopicDescription> retrievedTopicDescriptions = partitionMonitorClient
      .describeTopics(matchedTopicSet, describeTopicsOptions).all()
      .get(requestTimeoutMs - (System.currentTimeMillis() - startWait), TimeUnit.MILLISECONDS);
  return retrievedTopicDescriptions.values().stream()
      .map(topicDescription -> topicDescription.partitions().stream()
          .map(partitionInfo -> new LeaderTopicPartition(partitionInfo.leader().id(), topicDescription.name(),
              partitionInfo.partition())))
      .flatMap(Function.identity()).collect(Collectors.toSet());
}
 

/**
 * Returns the set of all topics in the Kafka cluster
 *
 * @return unmodifiable set of all topics in the Kafka cluster
 *
 * @throws AdminOperationException
 *      if there is an issue retrieving the set of all topics
 */
public Set<String> getTopics() {
    LOG.debug("Retrieving all topics");
    try {
        Set<String> topics = getNewAdminClient()
            .listTopics(new ListTopicsOptions().listInternal(true))
            .names().get(operationTimeout, TimeUnit.MILLISECONDS);
        if (topics.isEmpty()) {
            LOG.warn("Unable to list Kafka topics");
        }
        return Collections.unmodifiableSet(topics);
    } catch (InterruptedException | ExecutionException | TimeoutException e) {
        throw new AdminOperationException("Unable to list Kafka topics", e);
    }
}
 

protected Future<Set<String>> topicNames() {
    Promise<Set<String>> namesPromise = Promise.promise();
    ac.listTopics(new ListTopicsOptions().listInternal(true)).names()
            .whenComplete((names, error) -> {
                if (error != null) {
                    namesPromise.fail(error);
                } else {
                    log.debug("Got {} topic names", names.size());
                    namesPromise.complete(names);
                }
            });
    return namesPromise.future();
}
 

@Override
public Future<Set<String>> listTopics() {
    try {
        LOGGER.debug("Listing topics");
        ListTopicsOptions listOptions = new ListTopicsOptions().listInternal(true);
        return mapFuture(adminClient.listTopics(listOptions).names());
    } catch (Exception e) {
        return Future.failedFuture(e);
    }
}