Java Code Examples for org.apache.kafka.clients.admin.AdminClient#listConsumerGroupOffsets()

The following examples show how to use org.apache.kafka.clients.admin.AdminClient#listConsumerGroupOffsets() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: KafkaServiceImpl.java    From kafka-eagle with Apache License 2.0 5 votes vote down vote up 
/**
 * Get kafka 0.10.x, 1.x, 2.x offset from topic.
 */
public String getKafkaOffset(String clusterAlias) {
	Properties prop = new Properties();
	prop.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, parseBrokerServer(clusterAlias));

	if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) {
		sasl(prop, clusterAlias);
	}
	if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) {
		ssl(prop, clusterAlias);
	}
	JSONArray targets = new JSONArray();
	AdminClient adminClient = null;
	try {
		adminClient = AdminClient.create(prop);
		ListConsumerGroupsResult consumerGroups = adminClient.listConsumerGroups();
		java.util.Iterator<ConsumerGroupListing> groups = consumerGroups.all().get().iterator();
		while (groups.hasNext()) {
			String groupId = groups.next().groupId();
			if (!groupId.contains("kafka.eagle")) {
				ListConsumerGroupOffsetsResult offsets = adminClient.listConsumerGroupOffsets(groupId);
				for (Entry<TopicPartition, OffsetAndMetadata> entry : offsets.partitionsToOffsetAndMetadata().get().entrySet()) {
					JSONObject object = new JSONObject();
					object.put("group", groupId);
					object.put("topic", entry.getKey().topic());
					object.put("partition", entry.getKey().partition());
					object.put("offset", entry.getValue().offset());
					object.put("timestamp", CalendarUtils.getDate());
					targets.add(object);
				}
			}
		}
	} catch (Exception e) {
		LOG.error("Get consumer offset has error, msg is " + e.getMessage());
		e.printStackTrace();
	} finally {
		adminClient.close();
	}
	return targets.toJSONString();
}
Example 2
Source File: KafkaServiceImpl.java    From kafka-eagle with Apache License 2.0 5 votes vote down vote up 
/**
 * Get the data for the topic partition in the specified consumer group
 */
public Map<Integer, Long> getKafkaOffset(String clusterAlias, String group, String topic, Set<Integer> partitionids) {
	Map<Integer, Long> partitionOffset = new HashMap<>();
	Properties prop = new Properties();
	prop.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, parseBrokerServer(clusterAlias));

	if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) {
		sasl(prop, clusterAlias);
	}
	if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) {
		ssl(prop, clusterAlias);
	}
	AdminClient adminClient = null;
	try {
		adminClient = AdminClient.create(prop);
		List<TopicPartition> tps = new ArrayList<>();
		for (int partitionid : partitionids) {
			TopicPartition tp = new TopicPartition(topic, partitionid);
			tps.add(tp);
		}

		ListConsumerGroupOffsetsOptions consumerOffsetOptions = new ListConsumerGroupOffsetsOptions();
		consumerOffsetOptions.topicPartitions(tps);

		ListConsumerGroupOffsetsResult offsets = adminClient.listConsumerGroupOffsets(group, consumerOffsetOptions);
		for (Entry<TopicPartition, OffsetAndMetadata> entry : offsets.partitionsToOffsetAndMetadata().get().entrySet()) {
			if (topic.equals(entry.getKey().topic())) {
				partitionOffset.put(entry.getKey().partition(), entry.getValue().offset());
			}
		}
	} catch (Exception e) {
		LOG.error("Get consumer offset has error, msg is " + e.getMessage());
		e.printStackTrace();
	} finally {
		adminClient.close();
	}
	return partitionOffset;
}
Example 3
Source File: KafkaServiceImpl.java    From kafka-eagle with Apache License 2.0 5 votes vote down vote up 
/**
 * Get kafka group consumer all topics lags.
 */
public long getKafkaLag(String clusterAlias, String group, String ketopic) {
	long lag = 0L;

	Properties prop = new Properties();
	prop.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, parseBrokerServer(clusterAlias));

	if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) {
		sasl(prop, clusterAlias);
	}
	if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) {
		ssl(prop, clusterAlias);
	}
	AdminClient adminClient = null;
	try {
		adminClient = AdminClient.create(prop);
		ListConsumerGroupOffsetsResult offsets = adminClient.listConsumerGroupOffsets(group);
		for (Entry<TopicPartition, OffsetAndMetadata> entry : offsets.partitionsToOffsetAndMetadata().get().entrySet()) {
			if (ketopic.equals(entry.getKey().topic())) {
				long logSize = getKafkaLogSize(clusterAlias, entry.getKey().topic(), entry.getKey().partition());
				lag += logSize - entry.getValue().offset();
			}
		}
	} catch (Exception e) {
		LOG.error("Get cluster[" + clusterAlias + "] group[" + group + "] topic[" + ketopic + "] consumer lag has error, msg is " + e.getMessage());
		e.printStackTrace();
	} finally {
		adminClient.close();
	}
	return lag;
}