org.apache.kafka.clients.admin.ConsumerGroupListing Java Examples

The following examples show how to use org.apache.kafka.clients.admin.ConsumerGroupListing. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: OffsetSink.java    From kafka-backup with Apache License 2.0 5 votes vote down vote up 
public void syncConsumerGroups() {
    try {
        consumerGroups = adminClient.listConsumerGroups().all().get().stream().map(ConsumerGroupListing::groupId).collect(Collectors.toList());
    } catch (InterruptedException | ExecutionException e) {
        throw new RetriableException(e);
    }
}
Example #2
Source File: KafkaServiceImpl.java    From kafka-eagle with Apache License 2.0 5 votes vote down vote up 
/**
 * Get kafka 0.10.x, 1.x, 2.x consumer groups.
 */
public int getKafkaConsumerGroups(String clusterAlias) {
	Properties prop = new Properties();
	int counter = 0;
	prop.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, parseBrokerServer(clusterAlias));

	if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) {
		sasl(prop, clusterAlias);
	}
	if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) {
		ssl(prop, clusterAlias);
	}

	AdminClient adminClient = null;
	try {
		adminClient = AdminClient.create(prop);
		ListConsumerGroupsResult consumerGroups = adminClient.listConsumerGroups();
		java.util.Iterator<ConsumerGroupListing> groups = consumerGroups.all().get().iterator();
		while (groups.hasNext()) {
			String groupId = groups.next().groupId();
			if (!groupId.contains("kafka.eagle")) {
				counter++;
			}
		}
	} catch (Exception e) {
		LOG.info("Get kafka consumer group has error, msg is " + e.getMessage());
		e.printStackTrace();
	} finally {
		adminClient.close();
	}
	return counter;
}
Example #3
Source File: KafkaServiceImpl.java    From kafka-eagle with Apache License 2.0 5 votes vote down vote up 
/**
 * Get kafka 0.10.x, 1.x, 2.x offset from topic.
 */
public String getKafkaOffset(String clusterAlias) {
	Properties prop = new Properties();
	prop.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, parseBrokerServer(clusterAlias));

	if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) {
		sasl(prop, clusterAlias);
	}
	if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) {
		ssl(prop, clusterAlias);
	}
	JSONArray targets = new JSONArray();
	AdminClient adminClient = null;
	try {
		adminClient = AdminClient.create(prop);
		ListConsumerGroupsResult consumerGroups = adminClient.listConsumerGroups();
		java.util.Iterator<ConsumerGroupListing> groups = consumerGroups.all().get().iterator();
		while (groups.hasNext()) {
			String groupId = groups.next().groupId();
			if (!groupId.contains("kafka.eagle")) {
				ListConsumerGroupOffsetsResult offsets = adminClient.listConsumerGroupOffsets(groupId);
				for (Entry<TopicPartition, OffsetAndMetadata> entry : offsets.partitionsToOffsetAndMetadata().get().entrySet()) {
					JSONObject object = new JSONObject();
					object.put("group", groupId);
					object.put("topic", entry.getKey().topic());
					object.put("partition", entry.getKey().partition());
					object.put("offset", entry.getValue().offset());
					object.put("timestamp", CalendarUtils.getDate());
					targets.add(object);
				}
			}
		}
	} catch (Exception e) {
		LOG.error("Get consumer offset has error, msg is " + e.getMessage());
		e.printStackTrace();
	} finally {
		adminClient.close();
	}
	return targets.toJSONString();
}
Example #4
Source File: ApiControllerTest.java    From kafka-webview with MIT License 4 votes vote down vote up 
/**
 * Test the remove Consumer end point with admin role.
 */
@Test
@Transactional
public void test_removeConsumer_withAdminRole() throws Exception {
    // Create a cluster.
    final Cluster cluster = clusterTestTools.createCluster(
        "Test Cluster",
        sharedKafkaTestResource.getKafkaConnectString()
    );

    // Create a consumer with state on the cluster.
    final String consumerId = createConsumerWithState(new String[] {"TestTopic-" + System.currentTimeMillis()});

    // Construct payload
    final String payload = "{ \"consumerId\": \"" + consumerId + "\", \"clusterId\": \"" + cluster.getId() + "\"}";

    // Hit end point
    mockMvc
        .perform(post("/api/cluster/" + cluster.getId() + "/consumer/remove")
            .with(user(adminUserDetails))
            .with(csrf())
            .content(payload)
            .contentType(MediaType.APPLICATION_JSON)
        )
        .andDo(print())
        .andExpect(status().isOk())

        // Validate submit button seems to show up.
        .andExpect(content().string(containsString("true")));

    // Verify consumer no longer exists
    try (final AdminClient adminClient = sharedKafkaTestResource
        .getKafkaTestUtils()
        .getAdminClient()) {

        final ListConsumerGroupsResult request = adminClient.listConsumerGroups();
        final Collection<ConsumerGroupListing> results = request.all().get();

        final Optional<ConsumerGroupListing> match = results.stream()
            .filter((entry) -> (entry.groupId().equals(consumerId)))
            .findFirst();

        assertFalse("Should not have found entry", match.isPresent());
    }
}
Example #5
Source File: ApiControllerTest.java    From kafka-webview with MIT License 4 votes vote down vote up 
/**
 * Test the remove Consumer end point with non admin role.
 */
@Test
@Transactional
public void test_removeConsumer_withNonAdminRole() throws Exception {
    // Create a cluster.
    final Cluster cluster = clusterTestTools.createCluster(
        "Test Cluster",
        sharedKafkaTestResource.getKafkaConnectString()
    );

    // Create a consumer with state on the cluster.
    final String consumerId = createConsumerWithState(new String[] {"TestTopic-" + System.currentTimeMillis()});

    // Construct payload
    final String payload = "{ \"consumerId\": \"" + consumerId + "\", \"clusterId\": \"" + cluster.getId() + "\"}";

    // Hit end point
    mockMvc
        .perform(post("/api/cluster/" + cluster.getId() + "/consumer/remove")
            .with(user(nonAdminUserDetails))
            .with(csrf())
            .content(payload)
            .contentType(MediaType.APPLICATION_JSON)
        )
        .andDo(print())
        .andExpect(status().is4xxClientError());

    // Verify consumer still exists
    try (final AdminClient adminClient = sharedKafkaTestResource
        .getKafkaTestUtils()
        .getAdminClient()) {

        final ListConsumerGroupsResult request = adminClient.listConsumerGroups();
        final Collection<ConsumerGroupListing> results = request.all().get();

        final Optional<ConsumerGroupListing> match = results.stream()
            .filter((entry) -> (entry.groupId().equals(consumerId)))
            .findFirst();

        assertTrue("Should have found entry", match.isPresent());
    }
}