Java Code Examples for org.apache.kafka.common.internals.Topic#GROUP_METADATA_TOPIC_NAME

The following examples show how to use org.apache.kafka.common.internals.Topic#GROUP_METADATA_TOPIC_NAME . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: KafkaProtocolHandler.java    From kop with Apache License 2.0 6 votes vote down vote up 
private void loadOffsetTopics(GroupCoordinator groupCoordinator) throws Exception {
    String offsetsTopic = kafkaConfig.getKafkaMetadataTenant() + "/" + kafkaConfig.getKafkaMetadataNamespace()
        + "/" + Topic.GROUP_METADATA_TOPIC_NAME;
    int numPartitions = kafkaConfig.getOffsetsTopicNumPartitions();
    List<CompletableFuture<Void>> lists = Lists.newArrayListWithExpectedSize(numPartitions);
    for (int i = 0; i < numPartitions; i++) {
        String partition = offsetsTopic + PARTITIONED_TOPIC_SUFFIX + i;
        String broker = brokerService.pulsar().getAdminClient().lookups()
            .lookupTopic(partition);

        if (log.isDebugEnabled()) {
            log.debug("found broker {} for offset topic partition {}. current broker: {}",
                broker, partition, brokerService.pulsar().getBrokerServiceUrl());
        }

        if (broker.equalsIgnoreCase(brokerService.pulsar().getBrokerServiceUrl())) {
            lists.add(groupCoordinator.handleGroupImmigration(i));
        }
    }
    FutureUtil.waitForAll(lists).get();
}
Example 2
Source File: KafkaProtocolHandler.java    From kop with Apache License 2.0 5 votes vote down vote up 
private String createKafkaOffsetsTopic(BrokerService service) throws PulsarServerException, PulsarAdminException {
    String offsetsTopic = kafkaConfig.getKafkaMetadataTenant() + "/" + kafkaConfig.getKafkaMetadataNamespace()
        + "/" + Topic.GROUP_METADATA_TOPIC_NAME;

    PartitionedTopicMetadata offsetsTopicMetadata =
        service.pulsar().getAdminClient().topics().getPartitionedTopicMetadata(offsetsTopic);
    if (offsetsTopicMetadata.partitions <= 0) {
        log.info("Kafka group metadata topic {} doesn't exist. Creating it ...",
                offsetsTopic);
        try {
            service.pulsar().getAdminClient().topics().createPartitionedTopic(
                    offsetsTopic,
                    kafkaConfig.getOffsetsTopicNumPartitions()
            );

            for (int i = 0; i < kafkaConfig.getOffsetsTopicNumPartitions(); i++) {
                service.pulsar().getAdminClient().topics()
                        .createNonPartitionedTopic(offsetsTopic + PARTITIONED_TOPIC_SUFFIX + i);
            }
        } catch (ConflictException e) {
            log.info("Topic {} concurrent creating and cause e: ", offsetsTopic, e);
            return offsetsTopic;
        }

        log.info("Successfully created group metadata topic {} with {} partitions.",
                offsetsTopic, kafkaConfig.getOffsetsTopicNumPartitions());
    }

    return offsetsTopic;
}
Example 3
Source File: KafkaRequestHandler.java    From kop with Apache License 2.0 5 votes vote down vote up 
private boolean isOffsetTopic(String topic) {
    String offsetsTopic = kafkaConfig.getKafkaMetadataTenant() + "/"
        + kafkaConfig.getKafkaMetadataNamespace()
        + "/" + Topic.GROUP_METADATA_TOPIC_NAME;

    return topic.contains(offsetsTopic);
}
Example 4
Source File: GroupCoordinatorTest.java    From kop with Apache License 2.0 5 votes vote down vote up 
@Test
public void testCommitAndFetchOffsetsWithEmptyGroup() throws Exception {
    // For backwards compatibility, the coordinator supports committing/fetching offsets with an empty groupId.
    // To allow inspection and removal of the empty group, we must also support DescribeGroups and DeleteGroups

    TopicPartition tp = new TopicPartition("topic", 0);
    OffsetAndMetadata offset = OffsetAndMetadata.apply(0);
    String groupId = "";

    Map<TopicPartition, Errors> commitOffsetResult = groupCoordinator.handleCommitOffsets(
        groupId, OffsetCommitRequest.DEFAULT_MEMBER_ID, OffsetCommitRequest.DEFAULT_GENERATION_ID,
        ImmutableMap.<TopicPartition, OffsetAndMetadata>builder()
            .put(tp, offset)
            .build()
    ).get();
    assertEquals(Errors.NONE, commitOffsetResult.get(tp));

    KeyValue<Errors, Map<TopicPartition, PartitionData>> fetchOffsetsResult =
        groupCoordinator.handleFetchOffsets(groupId, Optional.of(Lists.newArrayList(tp)));
    assertEquals(Errors.NONE, fetchOffsetsResult.getKey());
    assertEquals(0, fetchOffsetsResult.getValue().get(tp).offset);

    KeyValue<Errors, GroupSummary> describeGroupResult = groupCoordinator.handleDescribeGroup(groupId);
    assertEquals(Errors.NONE, describeGroupResult.getKey());
    assertEquals(GroupState.Empty.toString(), describeGroupResult.getValue().state());

    TopicPartition groupTopicPartition = new TopicPartition(
        Topic.GROUP_METADATA_TOPIC_NAME, groupPartitionId
    );

    Map<String, Errors> deleteErrors = groupCoordinator.handleDeleteGroups(Sets.newHashSet(groupId));
    assertEquals(Errors.NONE, deleteErrors.get(groupId));

    KeyValue<Errors, Map<TopicPartition, PartitionData>> fetchOffsetsResult2 =
        groupCoordinator.handleFetchOffsets(groupId, Optional.of(Lists.newArrayList(tp)));
    assertEquals(Errors.NONE, fetchOffsetsResult2.getKey());
    assertEquals(OffsetFetchResponse.INVALID_OFFSET, fetchOffsetsResult2.getValue().get(tp).offset);
}
Example 5
Source File: GroupCoordinatorTest.java    From kop with Apache License 2.0 4 votes vote down vote up 
@Test
public void testBasicFetchTxnOffsets() throws Exception {
    TopicPartition tp = new TopicPartition("topic", 0);
    OffsetAndMetadata offset = OffsetAndMetadata.apply(0);
    long producerId = 1000L;
    short producerEpoch = 2;

    Map<TopicPartition, Errors> commitOffsetResult = groupCoordinator.handleTxnCommitOffsets(
        groupId, producerId, producerEpoch,
        ImmutableMap.<TopicPartition, OffsetAndMetadata>builder()
            .put(tp, offset)
            .build()
    ).get();
    assertEquals(Errors.NONE, commitOffsetResult.get(tp));

    KeyValue<Errors, Map<TopicPartition, PartitionData>> fetchOffsetsResult = groupCoordinator.handleFetchOffsets(
        groupId, Optional.of(Lists.newArrayList(tp))
    );

    // Validate that the offset isn't materialized yet.
    assertEquals(Errors.NONE, fetchOffsetsResult.getKey());
    assertEquals(OffsetFetchResponse.INVALID_OFFSET, fetchOffsetsResult.getValue().get(tp).offset);

    TopicPartition offsetsTopic = new TopicPartition(
        Topic.GROUP_METADATA_TOPIC_NAME,
        groupPartitionId
    );

    // send commit marker
    groupCoordinator.scheduleHandleTxnCompletion(
        producerId,
        Lists.newArrayList(offsetsTopic).stream(),
        TransactionResult.COMMIT
    ).get();

    // validate that committed offset is materialized
    KeyValue<Errors, Map<TopicPartition, PartitionData>> offsetFetchResult = groupCoordinator.handleFetchOffsets(
        groupId, Optional.of(Lists.newArrayList(tp))
    );
    assertEquals(Errors.NONE, offsetFetchResult.getKey());
    assertEquals(0, offsetFetchResult.getValue().get(tp).offset);
}
Example 6
Source File: GroupCoordinatorTest.java    From kop with Apache License 2.0 4 votes vote down vote up 
@Test
public void testFetchTxnOffsetsWithAbort() throws Exception {
    TopicPartition tp = new TopicPartition("topic", 0);
    OffsetAndMetadata offset = OffsetAndMetadata.apply(0);
    long producerId = 1000L;
    short producerEpoch = 2;

    Map<TopicPartition, Errors> commitOffsetResult = groupCoordinator.handleTxnCommitOffsets(
        groupId, producerId, producerEpoch,
        ImmutableMap.<TopicPartition, OffsetAndMetadata>builder()
            .put(tp, offset)
            .build()
    ).get();
    assertEquals(Errors.NONE, commitOffsetResult.get(tp));

    KeyValue<Errors, Map<TopicPartition, PartitionData>> fetchOffsetsResult = groupCoordinator.handleFetchOffsets(
        groupId, Optional.of(Lists.newArrayList(tp))
    );

    // Validate that the offset isn't materialized yet.
    assertEquals(Errors.NONE, fetchOffsetsResult.getKey());
    assertEquals(OffsetFetchResponse.INVALID_OFFSET, fetchOffsetsResult.getValue().get(tp).offset);

    TopicPartition offsetsTopic = new TopicPartition(
        Topic.GROUP_METADATA_TOPIC_NAME,
        groupPartitionId
    );

    // send commit marker
    groupCoordinator.scheduleHandleTxnCompletion(
        producerId,
        Lists.newArrayList(offsetsTopic).stream(),
        TransactionResult.ABORT
    ).get();

    KeyValue<Errors, Map<TopicPartition, PartitionData>> offsetFetchResult = groupCoordinator.handleFetchOffsets(
        groupId, Optional.of(Lists.newArrayList(tp))
    );
    assertEquals(Errors.NONE, offsetFetchResult.getKey());
    assertEquals(OffsetFetchResponse.INVALID_OFFSET, offsetFetchResult.getValue().get(tp).offset);
}
Example 7
Source File: GroupCoordinatorTest.java    From kop with Apache License 2.0 4 votes vote down vote up 
@Test
public void testFetchTxnOffsetsIgnoreSpuriousCommit() throws Exception {
    TopicPartition tp = new TopicPartition("topic", 0);
    OffsetAndMetadata offset = OffsetAndMetadata.apply(0);
    long producerId = 1000L;
    short producerEpoch = 2;

    Map<TopicPartition, Errors> commitOffsetResult = groupCoordinator.handleTxnCommitOffsets(
        groupId, producerId, producerEpoch,
        ImmutableMap.<TopicPartition, OffsetAndMetadata>builder()
            .put(tp, offset)
            .build()
    ).get();
    assertEquals(Errors.NONE, commitOffsetResult.get(tp));

    KeyValue<Errors, Map<TopicPartition, PartitionData>> fetchOffsetsResult = groupCoordinator.handleFetchOffsets(
        groupId, Optional.of(Lists.newArrayList(tp))
    );

    // Validate that the offset isn't materialized yet.
    assertEquals(Errors.NONE, fetchOffsetsResult.getKey());
    assertEquals(OffsetFetchResponse.INVALID_OFFSET, fetchOffsetsResult.getValue().get(tp).offset);

    TopicPartition offsetsTopic = new TopicPartition(
        Topic.GROUP_METADATA_TOPIC_NAME,
        groupPartitionId
    );

    // send commit marker
    groupCoordinator.scheduleHandleTxnCompletion(
        producerId,
        Lists.newArrayList(offsetsTopic).stream(),
        TransactionResult.ABORT
    ).get();

    KeyValue<Errors, Map<TopicPartition, PartitionData>> offsetFetchResult = groupCoordinator.handleFetchOffsets(
        groupId, Optional.of(Lists.newArrayList(tp))
    );
    assertEquals(Errors.NONE, offsetFetchResult.getKey());
    assertEquals(OffsetFetchResponse.INVALID_OFFSET, offsetFetchResult.getValue().get(tp).offset);

    // ignore spurious commit
    groupCoordinator.scheduleHandleTxnCompletion(
        producerId,
        Lists.newArrayList(offsetsTopic).stream(),
        TransactionResult.COMMIT
    ).get();

    KeyValue<Errors, Map<TopicPartition, PartitionData>> offsetFetchResult2 = groupCoordinator.handleFetchOffsets(
        groupId, Optional.of(Lists.newArrayList(tp))
    );
    assertEquals(Errors.NONE, offsetFetchResult2.getKey());
    assertEquals(OffsetFetchResponse.INVALID_OFFSET, offsetFetchResult2.getValue().get(tp).offset);
}
Example 8
Source File: GroupCoordinatorTest.java    From kop with Apache License 2.0 4 votes vote down vote up 
@Test
public void testFetchTxnOffsetsMultipleProducersOneGroup() throws Exception {
    // One group, two producers
    // Different producers will commit offsets for different partitions.
    // Each partition's offsets should be materialized when the corresponding producer's marker is received.
    List<TopicPartition> partitions = Lists.newArrayList(
        new TopicPartition("topic1", 0),
        new TopicPartition("topic2", 0)
    );
    List<OffsetAndMetadata> offsets = Lists.newArrayList(
        OffsetAndMetadata.apply(10),
        OffsetAndMetadata.apply(15)
    );

    List<Long> producerIds = Lists.newArrayList(1000L, 1005L);
    List<Short> producerEpochs = Lists.newArrayList((short) 3, (short) 4);

    TopicPartition offsetTopicPartition = new TopicPartition(
        Topic.GROUP_METADATA_TOPIC_NAME,
        groupMetadataManager.partitionFor(groupId)
    );

    List<Errors> errors = new ArrayList<>();
    List<Map<TopicPartition, PartitionData>> partitionData = new ArrayList<>();
    List<Map<TopicPartition, Errors>> commitOffsetResults = new ArrayList<>();

    // producer0 commits the offsets for partition0
    commitOffsetResults.add(
        groupCoordinator.handleTxnCommitOffsets(
            groupId, producerIds.get(0), producerEpochs.get(0),
            ImmutableMap.<TopicPartition, OffsetAndMetadata>builder()
                .put(partitions.get(0), offsets.get(0)).build()).get());
    assertEquals(Errors.NONE, commitOffsetResults.get(0).get(partitions.get(0)));

    // producer1 commits the offsets for partition1
    commitOffsetResults.add(
        groupCoordinator.handleTxnCommitOffsets(
            groupId, producerIds.get(1), producerEpochs.get(1),
            ImmutableMap.<TopicPartition, OffsetAndMetadata>builder()
                .put(partitions.get(1), offsets.get(1)).build()).get());
    assertEquals(Errors.NONE, commitOffsetResults.get(1).get(partitions.get(1)));

    // producer0 commits its transaction.
    groupCoordinator.scheduleHandleTxnCompletion(
        producerIds.get(0),
        Lists.newArrayList(offsetTopicPartition).stream(),
        TransactionResult.COMMIT
    ).get();

    KeyValue<Errors, Map<TopicPartition, PartitionData>> offsetFetchResult0 =
        groupCoordinator.handleFetchOffsets(groupId, Optional.of(partitions));
    errors.add(offsetFetchResult0.getKey());
    partitionData.add(offsetFetchResult0.getValue());

    assertEquals(Errors.NONE, errors.get(0));
    // we should only see the offset commit for producer0
    assertEquals(
        offsets.get(0).offset(),
        partitionData.get(0).get(partitions.get(0)).offset);
    assertEquals(
        OffsetFetchResponse.INVALID_OFFSET,
        partitionData.get(0).get(partitions.get(1)).offset);

    // producer 1 now commits its transaction
    groupCoordinator.scheduleHandleTxnCompletion(
        producerIds.get(1),
        Lists.newArrayList(offsetTopicPartition).stream(),
        TransactionResult.COMMIT
    ).get();

    KeyValue<Errors, Map<TopicPartition, PartitionData>> offsetFetchResult1 =
        groupCoordinator.handleFetchOffsets(groupId, Optional.of(partitions));
    errors.add(offsetFetchResult1.getKey());
    partitionData.add(offsetFetchResult1.getValue());

    assertEquals(Errors.NONE, errors.get(1));

    assertEquals(
        offsets.get(0).offset(),
        partitionData.get(1).get(partitions.get(0)).offset);
    assertEquals(
        offsets.get(1).offset(),
        partitionData.get(1).get(partitions.get(1)).offset);
}