Java Code Examples for org.apache.kafka.clients.consumer.KafkaConsumer#listTopics()

/**
 * Gets the total message count for the topic.
 * <b>WARNING: Don't use with compacted topics</b>
 */
@SuppressWarnings("unchecked")
public long getCount(String kafkaBrokers, String topic) {
    KafkaConsumer consumer = buildConsumer(kafkaBrokers);
    try {
        @SuppressWarnings("unchecked")
        Map<String, List<PartitionInfo>> topics = consumer.listTopics();
        List<PartitionInfo> partitionInfos = topics.get(topic);
        if (partitionInfos == null) {
            logger.warn("Partition information was not found for topic {}", topic);
            return 0;
        } else {
            Collection<TopicPartition> partitions = new ArrayList<>();
            for (PartitionInfo partitionInfo : partitionInfos) {
                TopicPartition partition = new TopicPartition(topic, partitionInfo.partition());
                partitions.add(partition);
            }
            Map<TopicPartition, Long> endingOffsets = consumer.endOffsets(partitions);
            Map<TopicPartition, Long> beginningOffsets = consumer.beginningOffsets(partitions);
            return diffOffsets(beginningOffsets, endingOffsets);
        }
    } finally {
        consumer.close();
    }
}
 

private boolean isKafkaConsumerOpen(KafkaConsumer consumer) {
    boolean ret = true;

    try {
        consumer.listTopics();
    } catch (IllegalStateException ex) {
        if (ex.getMessage().equalsIgnoreCase(consumerClosedErrorMsg)) {
            ret = false;
        }
    }

    return ret;
}
 

public boolean verifyTopicsExist(String kafkaBrokers, Set<String> requiredTopics,
                                 boolean checkPartitionCounts) {
    Properties props = new Properties();
    props.put("bootstrap.servers", kafkaBrokers);
    props.put("group.id", UUID.randomUUID().toString());
    props.put("key.deserializer", StringDeserializer.class.getName());
    props.put("value.deserializer", StringDeserializer.class.getName());
    if (serviceProperties != null) {
        SaslConfigurator configurator = new SaslConfigurator();
        configurator.configureSasl(props, serviceProperties.getServiceName(), serviceProperties.getKafkaPassword());
    } else {
        logger.warn("TopicVerification was not initialized, SASL will not be supported for this connection");
    }
    KafkaConsumer consumer = new KafkaConsumer(props);
    try {
        @SuppressWarnings("unchecked")
        Map<String, List<PartitionInfo>> topics = consumer.listTopics();

        Set<Integer> partitionCount = new HashSet<>();
        for (String requiredTopic : requiredTopics) {
            List<PartitionInfo> partitions = topics.get(requiredTopic);
            if (partitions == null) {
                logger.info("Required kafka topic {} not present", requiredTopic);
                return false;
            }
            partitionCount.add(partitions.size());
        }
        if (checkPartitionCounts && partitionCount.size() > 1) {
            logger.warn("Partition count mismatch in topics {}",
                    Arrays.toString(requiredTopics.toArray()));
            return false;
        }
        return true;
    } finally {
        consumer.close();
    }
}
 

/**
 * When an object implementing interface <code>Runnable</code> is used
 * to create a thread, starting the thread causes the object's
 * <code>run</code> method to be called in that separately executing
 * thread.
 * <p>
 * The general contract of the method <code>run</code> is that it may
 * take any action whatsoever.
 *
 * @see Thread#run()
 */
@Override
public void run() {
    String group = "kafka-insight-logOffsetListener";
    int sleepTime = 60000;
    KafkaConsumer<Array<Byte>, Array<Byte>> kafkaConsumer = null;

    while (true) {

        try {
            if (null == kafkaConsumer) {
                kafkaConsumer = KafkaUtils.createNewKafkaConsumer(brokersInfo, group);
            }

            Map<String, List<PartitionInfo>> topicPartitionsMap = kafkaConsumer.listTopics();
            for (List<PartitionInfo> partitionInfoList : topicPartitionsMap.values()) {
                for (PartitionInfo partitionInfo : partitionInfoList) {
                    TopicPartition topicPartition = new TopicPartition(partitionInfo.topic(), partitionInfo.partition());
                    Collection<TopicPartition> topicPartitions = Arrays.asList(topicPartition);
                    kafkaConsumer.assign(topicPartitions);
                    kafkaConsumer.seekToEnd(topicPartitions);
                    Long logEndOffset = kafkaConsumer.position(topicPartition);
                    logEndOffsetMap.put(topicPartition, logEndOffset);
                }
            }

            Thread.sleep(sleepTime);

        } catch (Exception e) {
            e.printStackTrace();
            if (null != kafkaConsumer) {
                kafkaConsumer.close();
                kafkaConsumer = null;
            }
        }
    }

}
 

/**
 * list topic test
 * @param consumer
 */
private static void listTopics(KafkaConsumer<Integer, String> consumer) {
    Map<String, List<PartitionInfo>> partitionInfo = consumer.listTopics();
    partitionInfo.forEach((String key,List<PartitionInfo> list)->{
        System.out.println("key = [" + key + "]");

        list.forEach((PartitionInfo partition)->{
           // partition.topic();
            System.out.println("PartitionInfo = [" + partition + "]");
        });
    });
}
 

/**
 * 按上次记录重置offsets
 */
private void resetCorrectOffsets(ConsumerWorker worker) {	
	
	KafkaConsumer<String, Serializable> consumer = worker.consumer;
	Map<String, List<PartitionInfo>> topicInfos = consumer.listTopics();
	Set<String> topics = topicInfos.keySet();
	
	List<String> expectTopics = new ArrayList<>(topicHandlers.keySet());
	
	List<PartitionInfo> patitions = null;
	
	consumer.poll(200);
	
	for (String topic : topics) {
		if(!expectTopics.contains(topic))continue;
		
		patitions = topicInfos.get(topic);
		for (PartitionInfo partition : patitions) {
			try {						
				//期望的偏移
				long expectOffsets = consumerContext.getLatestProcessedOffsets(topic, partition.partition());
				//
				TopicPartition topicPartition = new TopicPartition(partition.topic(), partition.partition());
				OffsetAndMetadata metadata = consumer.committed(topicPartition);
				
				Set<TopicPartition> assignment = consumer.assignment();
				if(assignment.contains(topicPartition)){
					if(expectOffsets > 0 && expectOffsets < metadata.offset()){								
						consumer.seek(topicPartition, expectOffsets);
						//consumer.seekToBeginning(assignment);
				        logger.info(">>>>>>> seek Topic[{}] partition[{}] from {} to {}",topic, partition.partition(),metadata.offset(),expectOffsets);
					}
				}
			} catch (Exception e) {
				logger.warn("try seek topic["+topic+"] partition["+partition.partition()+"] offsets error");
			}
		}
	}
	consumer.resume(consumer.assignment());
}
 

private void checkTopicExists(final KafkaConsumer<String, String> consumer) {
    Map<String, List<PartitionInfo>> topics = consumer.listTopics();
    while (!topics.containsKey("test")) {
        LOG.warn("Required topic is not available, only {} present", topics.keySet());
        sleep();
        topics = consumer.listTopics();
    }
    LOG.warn("Available topics: {}", topics.keySet());
}
 

private void checkTopicsAvailableInCluster() {
    Properties props = KafkaSource.createConsumerConfig(bootstrapServers, groupID, optionalConfigs,
            isBinaryMessage, enableOffsetCommit);
    props.put("group.id", "test-consumer-group");
    KafkaConsumer<String, String> consumer = new KafkaConsumer<>(props);
    Map<String, List<PartitionInfo>> testTopicList = consumer.listTopics();
    boolean topicsAvailable = true;
    StringBuilder invalidTopics = new StringBuilder("");
    for (String topic : topics) {
        boolean topicAvailable = false;
        for (Map.Entry<String, List<PartitionInfo>> entry : testTopicList.entrySet()) {
            if (entry.getKey().equals(topic)) {
                topicAvailable = true;
            }
        }
        if (!topicAvailable) {
            topicsAvailable = false;
            if ("".equals(invalidTopics.toString())) {
                invalidTopics.append(topic);
            } else {
                invalidTopics.append(',').append(topic);
            }
            LOG.warn("Topic, " + topic + " is not available.");
        }
    }
    if (null != partitions && !(partitions.length == 1 && partitions[0].equals("0")) && !topicsAvailable) {
        String errorMessage = "Topic(s) " + invalidTopics + " aren't available. Topics won't be created "
                + "since there are partition numbers defined in the query.";
        LOG.error(errorMessage);
        throw new SiddhiAppRuntimeException("Topic(s) " + invalidTopics + " aren't available. "
                + "Topics won't be created since there "
                + "are partition numbers defined in the query.");
    } else if (!topicsAvailable) {
        if (siddhiAppContext.isTransportChannelCreationEnabled()) {
            LOG.warn("Topic(s) " + invalidTopics + " aren't available. "
                    + "These Topics will be created with the default partition.");
        } else {
            throw new SiddhiAppRuntimeException("Topic(s) " + invalidTopics + " creation failed. " +
                    "User has disabled topic creation by setting " +
                    SiddhiConstants.TRANSPORT_CHANNEL_CREATION_IDENTIFIER +
                    " property to false. Hence Siddhi App deployment will be aborted.");
        }
    }
}
 

private Map<String, List<PartitionInfo>> getTopicPartitionInfoMap() {
  KafkaConsumer kafkaConsumer = KafkaUtils.getKafkaConsumer(zkUrl, securityProtocol, consumerConfigs);
  Map<String, List<PartitionInfo>> topicPartitonInfoMap = kafkaConsumer.listTopics();
  return topicPartitonInfoMap;
}
 

private Map<String, List<PartitionInfo>> getAllTopics() {
    final KafkaConsumer<String, String> kafkaConsumer = kafkaHelper.createConsumer();
    return kafkaConsumer.listTopics();
}