org.apache.kafka.clients.admin.ListTopicsResult Java Examples

public Map<String, TopicDescription> getTopicAndDescriptions() throws Exception {

		try {
			// 查询topic
			ListTopicsOptions lto = new ListTopicsOptions();
			lto.timeoutMs(10 * 1000);
			ListTopicsResult ltr = adminClient.listTopics(lto);
			
			// 查询topic配置信息
			DescribeTopicsOptions dto = new DescribeTopicsOptions();
			dto.timeoutMs(15 * 1000);
			DescribeTopicsResult dtr = adminClient.describeTopics(ltr.names().get(), dto);
			return dtr.all().get();
			
		} catch (Exception e) {
			throw e;
		}
	}
 

/**
 * <p> Discover any currently active Kafka topics. Return a list of strings filled with any default topics that need to be created.
 * @param client
 * @return topics_to_create <code>List</code> of <code>Strings</code> with all the topic names that need to be created.
 *                           Returns <code>null</code> if none need to be created.
 */
private static List<String> discoverTopics(final AdminClient client, final List<String> topics_to_discover)
{
    final List<String> topics_to_create = new ArrayList<>();

    // Discover what topics currently exist.
    try
    {
        final ListTopicsResult res = client.listTopics();
        final KafkaFuture<Set<String>> topics = res.names();
        final Set<String> topic_names = topics.get();

        for (String topic : topics_to_discover)
        {
            if ( ! topic_names.contains(topic))
                topics_to_create.add(topic);
        }
    }
    catch (Exception ex)
    {
        logger.log(Level.WARNING, "Unable to list topics. Automatic topic detection failed.", ex);
    }

    return topics_to_create;
}
 

private Set<String> listTopicsOrThrowException() throws ExecutionException, InterruptedException {
    // Use a key store helper if a self signed certificate is provided
    KeyStoreHelper brokerKeyStoreHelper = certificate != null ? new KeyStoreHelper(certificate, "brokerCertificate").store() : null;
    Properties properties = getKafkaAdminClientConfiguration(brokers, transportProtocol, brokerKeyStoreHelper);
    try (AdminClient client = KafkaAdminClient.create(properties)) {
        ListTopicsResult topics = client.listTopics();
        return topics.names().get();
    } finally {
        if (brokerKeyStoreHelper != null) {
            // Clean up temporary resources used by key store
            boolean keystoreDeleted = brokerKeyStoreHelper.clean();
            if (!keystoreDeleted) {
                LOG.warn("Impossible to delete temporary keystore located at " + brokerKeyStoreHelper.getKeyStorePath());
            }
        }
    }
}
 

public Set<String> getMissingTopics(Collection<String> topicsToCheck)
        throws InterruptedException {
    HashSet<String> missing = new HashSet<>(topicsToCheck);

    try (AdminClient adminClient = AdminClient.create(adminClientConfig)) {
        ListTopicsResult topics = adminClient.listTopics();
        Set<String> topicNames = topics.names().get(10, TimeUnit.SECONDS);

        if (topicNames.containsAll(topicsToCheck)) {
            return Collections.emptySet();
        } else {
            missing.removeAll(topicNames);
        }
    } catch (ExecutionException | TimeoutException e) {
        LOGGER.error("Failed to get topic names from broker", e);
    }

    return missing;
}
 

@Test(enabled = false)
public void testListTopics() throws Exception {
    Map<String, Object> adminClientConfig = new HashMap<>();
    adminClientConfig.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, "api.local.getbux.com:32400,api.local.getbux.com:32401,api.local.getbux.com:32402");
    adminClientConfig.put(AdminClientConfig.CLIENT_ID_CONFIG, "test-adminClient");
    AdminClient adminClient = AdminClient.create(adminClientConfig);

    // ensure all the topics are created
    ListTopicsResult listTopicsResult = adminClient.listTopics();
    Map<String, TopicListing> availableTopics = listTopicsResult.namesToListings().get();

    assertNotNull(availableTopics);
}
 

@Override
public void beforeAll(@Observes @Initialized(ApplicationScoped.class) Object event) throws Exception {
    Properties properties = new Properties();
    properties.put(
            CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG,
            System.getProperty(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092")
    );
    properties.put("connections.max.idle.ms", 10000);
    properties.put("request.timeout.ms", 5000);
    try (AdminClient client = AdminClient.create(properties)) {
        ListTopicsResult topics = client.listTopics();
        Set<String> names = topics.names().get();
        log.info("Kafka is running - {} ...", names);
    }
}
 

public void checkKafkaReady(int timeout, int wait) throws Exception {
    boolean ready = false;

    for (int i = 0; (i <= timeout); i += wait) {
        try {
            ListTopicsResult topics = adminClient.listTopics();
            Set<String> names = topics.names().get();
            if (names.isEmpty()) {
                // case: if no topic found.
                logger.info("Kafka not ready yet to accept requests. Waiting...");
                Thread.sleep(wait * 1000);
            } else {
                ready = true;
                logger.info("Kafka ready to accept requests after " + i + " seconds.");
                break;
            }
        } catch (Exception e) {
            // Kafka is not available
            logger.info("Kafka not ready yet to accept requests. Waiting...");
            Thread.sleep(wait * 1000);
        }
    }

    if (!ready) {
        throw new Exception("Kafka not ready after " + timeout + " seconds.");
    }
}
 

ListTopicsResult mockListTopics() {
    ListTopicsResult ltr = mock(ListTopicsResult.class);
    when(ltr.names()).thenAnswer(invocation -> {
        return listTopicsResult != null ? failedFuture(listTopicsResult) : KafkaFuture.completedFuture(new HashSet<>(topics.keySet()));
    });
    when(ltr.listings()).thenThrow(notImplemented());
    when(ltr.namesToListings()).thenThrow(notImplemented());
    return ltr;
}
 

@Override
public Future<Set<String>> listTopics() {
  ContextInternal ctx = (ContextInternal) vertx.getOrCreateContext();
  Promise<Set<String>> promise = ctx.promise();

  ListTopicsResult listTopicsResult = this.adminClient.listTopics();
  listTopicsResult.names().whenComplete((topics, ex) -> {
    if (ex == null) {
      promise.complete(topics);
    } else {
      promise.fail(ex);
    }
  });
  return promise.future();
}
 

public Set<ClusterTopicInfo> describeTopics() throws InterruptedException, ExecutionException, TimeoutException {

        Set<ClusterTopicInfo> result = new HashSet<>();
        final ListTopicsResult listTopicsResult = kafkaClientsAdminClient.listTopics(new ListTopicsOptions().listInternal(false));
        final Collection<TopicListing> listings = listTopicsResult.listings().get(ApplicationConstants.FUTURE_GET_TIMEOUT_MS, TimeUnit.MILLISECONDS);
        Logger.debug(String.format("describeTopics.listings %s", listings));


        final Set<String> topicNames = listTopicsResult.names().get(ApplicationConstants.FUTURE_GET_TIMEOUT_MS, TimeUnit.MILLISECONDS);
        final DescribeTopicsResult describeTopicsResult = kafkaClientsAdminClient.describeTopics(topicNames);
        final Map<String, TopicDescription> stringTopicDescriptionMap = describeTopicsResult.all().get(ApplicationConstants.FUTURE_GET_TIMEOUT_MS,
                                                                                                       TimeUnit.MILLISECONDS);

        for (Map.Entry<String, TopicDescription> entry : stringTopicDescriptionMap.entrySet()) {
            final TopicDescription topicDescription = entry.getValue();
            final ClusterTopicInfo clusterTopicInfo = new ClusterTopicInfo(topicDescription.name(),
                                                                           topicDescription.partitions(),
                                                                           getConfigEntriesForTopic(topicDescription.name()));
            result.add(clusterTopicInfo);
        }
        return result;

    }
 

private ListTopicsResult getListTopicsResult() {
  ListTopicsResult listTopicsResult = mock(ListTopicsResult.class);
  List<String> topicNamesList = Arrays.asList(topicName1, topicName2, topicName3);
  expect(listTopicsResult.names())
      .andReturn(KafkaFuture.completedFuture(new HashSet<>(topicNamesList)));
  replay(listTopicsResult);
  return listTopicsResult;
}
 

private ListTopicsResult getListTopicsResultWithInternalTopics() {
  ListTopicsResult listTopicsResult = mock(ListTopicsResult.class);
  List<String> topicNamesList = Arrays.asList(topicName1, topicName2, topicName3,
                                              internalTopic1, internalTopic2,
                                              confluentInternalTopic);
  expect(listTopicsResult.names())
      .andReturn(KafkaFuture.completedFuture(new HashSet<>(topicNamesList)));
  replay(listTopicsResult);
  return listTopicsResult;
}
 

@SuppressWarnings("unchecked")
private ListTopicsResult listTopicResultWithNotControllerException() {
  ListTopicsResult listTopicsResult = mock(ListTopicsResult.class);
  expect(listTopicsResult.names())
      .andReturn(failedFuture(new NotControllerException("Not Controller")));
  replay(listTopicsResult);
  return listTopicsResult;
}
 

private ListTopicsResult getEmptyListTopicResult() {
  ListTopicsResult listTopicsResult = mock(ListTopicsResult.class);
  List<String> topicNamesList = Collections.emptyList();
  expect(listTopicsResult.names())
      .andReturn(KafkaFuture.completedFuture(new HashSet<>(topicNamesList)));
  replay(listTopicsResult);
  return listTopicsResult;
}
 

@Test
public void testListExisting() {
  Cluster cluster = createCluster(1);
  TopicPartitionInfo tp = new TopicPartitionInfo(0, cluster.nodeById(0), cluster.nodes(), Collections.emptyList());
  ConfigEntry configEntry = new ConfigEntry("k", "v");
  KafkaFuture<Config> kfc = KafkaFuture.completedFuture(new Config(Collections.singletonList(configEntry)));
  Set<String> topicNames = new HashSet<>(Arrays.asList("a", "b", "_c"));
  Map<String, TopicDescription> tds = new HashMap<String, TopicDescription>() {
    {
      put("a", new TopicDescription("a", false, Collections.singletonList(tp)));
      put("b", new TopicDescription("b", false, Collections.singletonList(tp)));
      put("c", new TopicDescription("_c", false, Collections.singletonList(tp)));
    }
  };
  Map<ConfigResource, KafkaFuture<Config>> configs = new HashMap<ConfigResource, KafkaFuture<Config>>() {
    {
      put(new ConfigResource(TOPIC, "a"), kfc);
      put(new ConfigResource(TOPIC, "b"), kfc);
      put(new ConfigResource(TOPIC, "_c"), kfc);
    }
  };

  TopicService service = new TopicServiceImpl(adminClient, true);
  ListTopicsResult listTopicsResult = mock(ListTopicsResult.class);
  DescribeTopicsResult describeTopicsResult = mock(DescribeTopicsResult.class);
  DescribeConfigsResult describeConfigsResult = mock(DescribeConfigsResult.class);

  when(describeTopicsResult.all()).thenReturn(KafkaFuture.completedFuture(tds));
  when(listTopicsResult.names()).thenReturn(KafkaFuture.completedFuture(topicNames));
  when(describeConfigsResult.values()).thenReturn(configs);
  when(adminClient.listTopics(any(ListTopicsOptions.class))).thenReturn(listTopicsResult);
  when(adminClient.describeTopics(topicNames)).thenReturn(describeTopicsResult);
  when(adminClient.describeConfigs(any(Collection.class))).thenReturn(describeConfigsResult);

  Map<String, ConfiguredTopic> actual = service.listExisting(true);
  Assert.assertEquals(2, actual.size());
  Assert.assertEquals(new HashSet<>(Arrays.asList("a", "b")), actual.keySet());
}
 

@Override
public void beforeAll(@Observes @Initialized(ApplicationScoped.class) Object event) throws Exception {
    Properties properties = new Properties();
    properties.put(
            CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG,
            System.getProperty(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092")
    );
    properties.put("connections.max.idle.ms", 10000);
    properties.put("request.timeout.ms", 5000);
    try (AdminClient client = AdminClient.create(properties)) {
        ListTopicsResult topics = client.listTopics();
        Set<String> names = topics.names().get();
        log.info("Kafka is running - {} ...", names);
    }
}
 

@Override
protected void doHealthCheck(Health.Builder builder) throws Exception {
	try {
		this.lock.lock();
		if (this.adminClient == null) {
			this.adminClient = AdminClient.create(this.adminClientProperties);
		}
		final Status status = healthStatusThreadLocal.get();
		//If one of the kafka streams binders (kstream, ktable, globalktable) was down before on the same request,
		//retrieve that from the thead local storage where it was saved before. This is done in order to avoid
		//the duration of the total health check since in the case of Kafka Streams each binder tries to do
		//its own health check and since we already know that this is DOWN, simply pass that information along.
		if (status == Status.DOWN) {
			builder.withDetail("No topic information available", "Kafka broker is not reachable");
			builder.status(Status.DOWN);
		}
		else {
			final ListTopicsResult listTopicsResult = this.adminClient.listTopics();
			listTopicsResult.listings().get(this.configurationProperties.getHealthTimeout(), TimeUnit.SECONDS);

			if (this.kafkaStreamsBindingInformationCatalogue.getStreamsBuilderFactoryBeans().isEmpty()) {
				builder.withDetail("No Kafka Streams bindings have been established", "Kafka Streams binder did not detect any processors");
				builder.status(Status.UNKNOWN);
			}
			else {
				boolean up = true;
				for (KafkaStreams kStream : kafkaStreamsRegistry.getKafkaStreams()) {
					up &= kStream.state().isRunningOrRebalancing();
					builder.withDetails(buildDetails(kStream));
				}
				builder.status(up ? Status.UP : Status.DOWN);
			}
		}
	}
	catch (Exception e) {
		builder.withDetail("No topic information available", "Kafka broker is not reachable");
		builder.status(Status.DOWN);
		builder.withException(e);
		//Store binder down status into a thread local storage.
		healthStatusThreadLocal.set(Status.DOWN);
	}
	finally {
		this.lock.unlock();
	}
}