org.apache.kafka.clients.consumer.internals.AbstractPartitionAssignor Java Examples

The following examples show how to use org.apache.kafka.clients.consumer.internals.AbstractPartitionAssignor. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: BroadcastAssignor.java    From kafka_book_demo with Apache License 2.0 5 votes vote down vote up 
@Override
public Map<String, List<TopicPartition>> assign(
        Map<String, Integer> partitionsPerTopic,
        Map<String, Subscription> subscriptions) {
    Map<String, List<String>> consumersPerTopic =
            consumersPerTopic(subscriptions);
    Map<String, List<TopicPartition>> assignment = new HashMap<>();
    //Java8
    subscriptions.keySet().forEach(memberId ->
            assignment.put(memberId, new ArrayList<>()));

    //针对每一个主题，为每一个订阅的消费者分配所有的分区
    consumersPerTopic.entrySet().forEach(topicEntry->{
        String topic = topicEntry.getKey();
        List<String> members = topicEntry.getValue();

        Integer numPartitionsForTopic = partitionsPerTopic.get(topic);
        if (numPartitionsForTopic == null || members.isEmpty())
            return;
        List<TopicPartition> partitions = AbstractPartitionAssignor
            .partitions(topic, numPartitionsForTopic);
        if (!partitions.isEmpty()) {
            members.forEach(memberId ->
                    assignment.get(memberId).addAll(partitions));
        }
    });
    return assignment;
}
Example #2
Source File: RandomAssignor.java    From kafka_book_demo with Apache License 2.0 5 votes vote down vote up 
@Override
public Map<String, List<TopicPartition>> assign(Map<String, Integer> partitionsPerTopic, Map<String, Subscription> subscriptions) {
    Map<String, List<String>> consumersPerTopic =
            consumersPerTopic(subscriptions);
    Map<String, List<TopicPartition>> assignment = new HashMap<>();
    for (String memberId : subscriptions.keySet()) {
        assignment.put(memberId, new ArrayList<>());
    }

    //针对每一个主题进行分区分配
    for (Map.Entry<String, List<String>> topicEntry :
            consumersPerTopic.entrySet()) {
        String topic = topicEntry.getKey();
        List<String> consumersForTopic = topicEntry.getValue();
        int consumerSize = consumersForTopic.size();
        Integer numPartitionsForTopic = partitionsPerTopic.get(topic);
        if (numPartitionsForTopic == null) {
            continue;
        }

        //当前主题下的所有分区
        List<TopicPartition> partitions =
                AbstractPartitionAssignor.partitions(topic,
                numPartitionsForTopic);
        //将每个分区随机分配给一个消费者
        for (TopicPartition partition : partitions) {
            int rand = new Random().nextInt(consumerSize);
            String randomConsumer = consumersForTopic.get(rand);
            assignment.get(randomConsumer).add(partition);
        }
    }
    return assignment;
}
Example #3
Source File: BroadcastAssignor.java    From BigData-In-Practice with Apache License 2.0 5 votes vote down vote up 
@Override
public Map<String, List<TopicPartition>> assign(
        Map<String, Integer> partitionsPerTopic,
        Map<String, Subscription> subscriptions) {
    Map<String, List<String>> consumersPerTopic = consumersPerTopic(subscriptions);
    Map<String, List<TopicPartition>> assignment = new HashMap<>();
    //Java8
    subscriptions.keySet().forEach(memberId ->
            assignment.put(memberId, new ArrayList<>()));

    //针对每一个主题，为每一个订阅的消费者分配所有的分区
    consumersPerTopic.entrySet().forEach(topicEntry -> {
        String topic = topicEntry.getKey();
        List<String> members = topicEntry.getValue();

        Integer numPartitionsForTopic = partitionsPerTopic.get(topic);
        if (numPartitionsForTopic == null || members.isEmpty())
            return;
        List<TopicPartition> partitions = AbstractPartitionAssignor
                .partitions(topic, numPartitionsForTopic);
        if (!partitions.isEmpty()) {
            members.forEach(memberId ->
                    assignment.get(memberId).addAll(partitions));
        }
    });
    return assignment;
}
Example #4
Source File: RandomAssignor.java    From BigData-In-Practice with Apache License 2.0 5 votes vote down vote up 
@Override
public Map<String, List<TopicPartition>> assign(Map<String, Integer> partitionsPerTopic, Map<String, Subscription> subscriptions) {
    Map<String, List<String>> consumersPerTopic = consumersPerTopic(subscriptions);
    Map<String, List<TopicPartition>> assignment = new HashMap<>();
    for (String memberId : subscriptions.keySet()) {
        assignment.put(memberId, new ArrayList<>());
    }

    //针对每一个主题进行分区分配
    for (Map.Entry<String, List<String>> topicEntry :
            consumersPerTopic.entrySet()) {
        String topic = topicEntry.getKey();
        List<String> consumersForTopic = topicEntry.getValue();
        int consumerSize = consumersForTopic.size();
        Integer numPartitionsForTopic = partitionsPerTopic.get(topic);
        if (numPartitionsForTopic == null) {
            continue;
        }

        //当前主题下的所有分区
        List<TopicPartition> partitions =
                AbstractPartitionAssignor.partitions(topic,
                        numPartitionsForTopic);
        //将每个分区随机分配给一个消费者
        for (TopicPartition partition : partitions) {
            int rand = new Random().nextInt(consumerSize);
            String randomConsumer = consumersForTopic.get(rand);
            assignment.get(randomConsumer).add(partition);
        }
    }
    return assignment;
}