Java Code Examples for org.apache.kafka.common.internals.Topic#GROUP_METADATA_TOPIC_NAME

private void loadOffsetTopics(GroupCoordinator groupCoordinator) throws Exception {
    String offsetsTopic = kafkaConfig.getKafkaMetadataTenant() + "/" + kafkaConfig.getKafkaMetadataNamespace()
        + "/" + Topic.GROUP_METADATA_TOPIC_NAME;
    int numPartitions = kafkaConfig.getOffsetsTopicNumPartitions();
    List<CompletableFuture<Void>> lists = Lists.newArrayListWithExpectedSize(numPartitions);
    for (int i = 0; i < numPartitions; i++) {
        String partition = offsetsTopic + PARTITIONED_TOPIC_SUFFIX + i;
        String broker = brokerService.pulsar().getAdminClient().lookups()
            .lookupTopic(partition);

        if (log.isDebugEnabled()) {
            log.debug("found broker {} for offset topic partition {}. current broker: {}",
                broker, partition, brokerService.pulsar().getBrokerServiceUrl());
        }

        if (broker.equalsIgnoreCase(brokerService.pulsar().getBrokerServiceUrl())) {
            lists.add(groupCoordinator.handleGroupImmigration(i));
        }
    }
    FutureUtil.waitForAll(lists).get();
}
 

private String createKafkaOffsetsTopic(BrokerService service) throws PulsarServerException, PulsarAdminException {
    String offsetsTopic = kafkaConfig.getKafkaMetadataTenant() + "/" + kafkaConfig.getKafkaMetadataNamespace()
        + "/" + Topic.GROUP_METADATA_TOPIC_NAME;

    PartitionedTopicMetadata offsetsTopicMetadata =
        service.pulsar().getAdminClient().topics().getPartitionedTopicMetadata(offsetsTopic);
    if (offsetsTopicMetadata.partitions <= 0) {
        log.info("Kafka group metadata topic {} doesn't exist. Creating it ...",
                offsetsTopic);
        try {
            service.pulsar().getAdminClient().topics().createPartitionedTopic(
                    offsetsTopic,
                    kafkaConfig.getOffsetsTopicNumPartitions()
            );

            for (int i = 0; i < kafkaConfig.getOffsetsTopicNumPartitions(); i++) {
                service.pulsar().getAdminClient().topics()
                        .createNonPartitionedTopic(offsetsTopic + PARTITIONED_TOPIC_SUFFIX + i);
            }
        } catch (ConflictException e) {
            log.info("Topic {} concurrent creating and cause e: ", offsetsTopic, e);
            return offsetsTopic;
        }

        log.info("Successfully created group metadata topic {} with {} partitions.",
                offsetsTopic, kafkaConfig.getOffsetsTopicNumPartitions());
    }

    return offsetsTopic;
}
 

private boolean isOffsetTopic(String topic) {
    String offsetsTopic = kafkaConfig.getKafkaMetadataTenant() + "/"
        + kafkaConfig.getKafkaMetadataNamespace()
        + "/" + Topic.GROUP_METADATA_TOPIC_NAME;

    return topic.contains(offsetsTopic);
}
 

@Test
public void testCommitAndFetchOffsetsWithEmptyGroup() throws Exception {
    // For backwards compatibility, the coordinator supports committing/fetching offsets with an empty groupId.
    // To allow inspection and removal of the empty group, we must also support DescribeGroups and DeleteGroups

    TopicPartition tp = new TopicPartition("topic", 0);
    OffsetAndMetadata offset = OffsetAndMetadata.apply(0);
    String groupId = "";

    Map<TopicPartition, Errors> commitOffsetResult = groupCoordinator.handleCommitOffsets(
        groupId, OffsetCommitRequest.DEFAULT_MEMBER_ID, OffsetCommitRequest.DEFAULT_GENERATION_ID,
        ImmutableMap.<TopicPartition, OffsetAndMetadata>builder()
            .put(tp, offset)
            .build()
    ).get();
    assertEquals(Errors.NONE, commitOffsetResult.get(tp));

    KeyValue<Errors, Map<TopicPartition, PartitionData>> fetchOffsetsResult =
        groupCoordinator.handleFetchOffsets(groupId, Optional.of(Lists.newArrayList(tp)));
    assertEquals(Errors.NONE, fetchOffsetsResult.getKey());
    assertEquals(0, fetchOffsetsResult.getValue().get(tp).offset);

    KeyValue<Errors, GroupSummary> describeGroupResult = groupCoordinator.handleDescribeGroup(groupId);
    assertEquals(Errors.NONE, describeGroupResult.getKey());
    assertEquals(GroupState.Empty.toString(), describeGroupResult.getValue().state());

    TopicPartition groupTopicPartition = new TopicPartition(
        Topic.GROUP_METADATA_TOPIC_NAME, groupPartitionId
    );

    Map<String, Errors> deleteErrors = groupCoordinator.handleDeleteGroups(Sets.newHashSet(groupId));
    assertEquals(Errors.NONE, deleteErrors.get(groupId));

    KeyValue<Errors, Map<TopicPartition, PartitionData>> fetchOffsetsResult2 =
        groupCoordinator.handleFetchOffsets(groupId, Optional.of(Lists.newArrayList(tp)));
    assertEquals(Errors.NONE, fetchOffsetsResult2.getKey());
    assertEquals(OffsetFetchResponse.INVALID_OFFSET, fetchOffsetsResult2.getValue().get(tp).offset);
}
 

@Test
public void testBasicFetchTxnOffsets() throws Exception {
    TopicPartition tp = new TopicPartition("topic", 0);
    OffsetAndMetadata offset = OffsetAndMetadata.apply(0);
    long producerId = 1000L;
    short producerEpoch = 2;

    Map<TopicPartition, Errors> commitOffsetResult = groupCoordinator.handleTxnCommitOffsets(
        groupId, producerId, producerEpoch,
        ImmutableMap.<TopicPartition, OffsetAndMetadata>builder()
            .put(tp, offset)
            .build()
    ).get();
    assertEquals(Errors.NONE, commitOffsetResult.get(tp));

    KeyValue<Errors, Map<TopicPartition, PartitionData>> fetchOffsetsResult = groupCoordinator.handleFetchOffsets(
        groupId, Optional.of(Lists.newArrayList(tp))
    );

    // Validate that the offset isn't materialized yet.
    assertEquals(Errors.NONE, fetchOffsetsResult.getKey());
    assertEquals(OffsetFetchResponse.INVALID_OFFSET, fetchOffsetsResult.getValue().get(tp).offset);

    TopicPartition offsetsTopic = new TopicPartition(
        Topic.GROUP_METADATA_TOPIC_NAME,
        groupPartitionId
    );

    // send commit marker
    groupCoordinator.scheduleHandleTxnCompletion(
        producerId,
        Lists.newArrayList(offsetsTopic).stream(),
        TransactionResult.COMMIT
    ).get();

    // validate that committed offset is materialized
    KeyValue<Errors, Map<TopicPartition, PartitionData>> offsetFetchResult = groupCoordinator.handleFetchOffsets(
        groupId, Optional.of(Lists.newArrayList(tp))
    );
    assertEquals(Errors.NONE, offsetFetchResult.getKey());
    assertEquals(0, offsetFetchResult.getValue().get(tp).offset);
}
 

@Test
public void testFetchTxnOffsetsWithAbort() throws Exception {
    TopicPartition tp = new TopicPartition("topic", 0);
    OffsetAndMetadata offset = OffsetAndMetadata.apply(0);
    long producerId = 1000L;
    short producerEpoch = 2;

    Map<TopicPartition, Errors> commitOffsetResult = groupCoordinator.handleTxnCommitOffsets(
        groupId, producerId, producerEpoch,
        ImmutableMap.<TopicPartition, OffsetAndMetadata>builder()
            .put(tp, offset)
            .build()
    ).get();
    assertEquals(Errors.NONE, commitOffsetResult.get(tp));

    KeyValue<Errors, Map<TopicPartition, PartitionData>> fetchOffsetsResult = groupCoordinator.handleFetchOffsets(
        groupId, Optional.of(Lists.newArrayList(tp))
    );

    // Validate that the offset isn't materialized yet.
    assertEquals(Errors.NONE, fetchOffsetsResult.getKey());
    assertEquals(OffsetFetchResponse.INVALID_OFFSET, fetchOffsetsResult.getValue().get(tp).offset);

    TopicPartition offsetsTopic = new TopicPartition(
        Topic.GROUP_METADATA_TOPIC_NAME,
        groupPartitionId
    );

    // send commit marker
    groupCoordinator.scheduleHandleTxnCompletion(
        producerId,
        Lists.newArrayList(offsetsTopic).stream(),
        TransactionResult.ABORT
    ).get();

    KeyValue<Errors, Map<TopicPartition, PartitionData>> offsetFetchResult = groupCoordinator.handleFetchOffsets(
        groupId, Optional.of(Lists.newArrayList(tp))
    );
    assertEquals(Errors.NONE, offsetFetchResult.getKey());
    assertEquals(OffsetFetchResponse.INVALID_OFFSET, offsetFetchResult.getValue().get(tp).offset);
}
 

@Test
public void testFetchTxnOffsetsIgnoreSpuriousCommit() throws Exception {
    TopicPartition tp = new TopicPartition("topic", 0);
    OffsetAndMetadata offset = OffsetAndMetadata.apply(0);
    long producerId = 1000L;
    short producerEpoch = 2;

    Map<TopicPartition, Errors> commitOffsetResult = groupCoordinator.handleTxnCommitOffsets(
        groupId, producerId, producerEpoch,
        ImmutableMap.<TopicPartition, OffsetAndMetadata>builder()
            .put(tp, offset)
            .build()
    ).get();
    assertEquals(Errors.NONE, commitOffsetResult.get(tp));

    KeyValue<Errors, Map<TopicPartition, PartitionData>> fetchOffsetsResult = groupCoordinator.handleFetchOffsets(
        groupId, Optional.of(Lists.newArrayList(tp))
    );

    // Validate that the offset isn't materialized yet.
    assertEquals(Errors.NONE, fetchOffsetsResult.getKey());
    assertEquals(OffsetFetchResponse.INVALID_OFFSET, fetchOffsetsResult.getValue().get(tp).offset);

    TopicPartition offsetsTopic = new TopicPartition(
        Topic.GROUP_METADATA_TOPIC_NAME,
        groupPartitionId
    );

    // send commit marker
    groupCoordinator.scheduleHandleTxnCompletion(
        producerId,
        Lists.newArrayList(offsetsTopic).stream(),
        TransactionResult.ABORT
    ).get();

    KeyValue<Errors, Map<TopicPartition, PartitionData>> offsetFetchResult = groupCoordinator.handleFetchOffsets(
        groupId, Optional.of(Lists.newArrayList(tp))
    );
    assertEquals(Errors.NONE, offsetFetchResult.getKey());
    assertEquals(OffsetFetchResponse.INVALID_OFFSET, offsetFetchResult.getValue().get(tp).offset);

    // ignore spurious commit
    groupCoordinator.scheduleHandleTxnCompletion(
        producerId,
        Lists.newArrayList(offsetsTopic).stream(),
        TransactionResult.COMMIT
    ).get();

    KeyValue<Errors, Map<TopicPartition, PartitionData>> offsetFetchResult2 = groupCoordinator.handleFetchOffsets(
        groupId, Optional.of(Lists.newArrayList(tp))
    );
    assertEquals(Errors.NONE, offsetFetchResult2.getKey());
    assertEquals(OffsetFetchResponse.INVALID_OFFSET, offsetFetchResult2.getValue().get(tp).offset);
}
 

@Test
public void testFetchTxnOffsetsMultipleProducersOneGroup() throws Exception {
    // One group, two producers
    // Different producers will commit offsets for different partitions.
    // Each partition's offsets should be materialized when the corresponding producer's marker is received.
    List<TopicPartition> partitions = Lists.newArrayList(
        new TopicPartition("topic1", 0),
        new TopicPartition("topic2", 0)
    );
    List<OffsetAndMetadata> offsets = Lists.newArrayList(
        OffsetAndMetadata.apply(10),
        OffsetAndMetadata.apply(15)
    );

    List<Long> producerIds = Lists.newArrayList(1000L, 1005L);
    List<Short> producerEpochs = Lists.newArrayList((short) 3, (short) 4);

    TopicPartition offsetTopicPartition = new TopicPartition(
        Topic.GROUP_METADATA_TOPIC_NAME,
        groupMetadataManager.partitionFor(groupId)
    );

    List<Errors> errors = new ArrayList<>();
    List<Map<TopicPartition, PartitionData>> partitionData = new ArrayList<>();
    List<Map<TopicPartition, Errors>> commitOffsetResults = new ArrayList<>();

    // producer0 commits the offsets for partition0
    commitOffsetResults.add(
        groupCoordinator.handleTxnCommitOffsets(
            groupId, producerIds.get(0), producerEpochs.get(0),
            ImmutableMap.<TopicPartition, OffsetAndMetadata>builder()
                .put(partitions.get(0), offsets.get(0)).build()).get());
    assertEquals(Errors.NONE, commitOffsetResults.get(0).get(partitions.get(0)));

    // producer1 commits the offsets for partition1
    commitOffsetResults.add(
        groupCoordinator.handleTxnCommitOffsets(
            groupId, producerIds.get(1), producerEpochs.get(1),
            ImmutableMap.<TopicPartition, OffsetAndMetadata>builder()
                .put(partitions.get(1), offsets.get(1)).build()).get());
    assertEquals(Errors.NONE, commitOffsetResults.get(1).get(partitions.get(1)));

    // producer0 commits its transaction.
    groupCoordinator.scheduleHandleTxnCompletion(
        producerIds.get(0),
        Lists.newArrayList(offsetTopicPartition).stream(),
        TransactionResult.COMMIT
    ).get();

    KeyValue<Errors, Map<TopicPartition, PartitionData>> offsetFetchResult0 =
        groupCoordinator.handleFetchOffsets(groupId, Optional.of(partitions));
    errors.add(offsetFetchResult0.getKey());
    partitionData.add(offsetFetchResult0.getValue());

    assertEquals(Errors.NONE, errors.get(0));
    // we should only see the offset commit for producer0
    assertEquals(
        offsets.get(0).offset(),
        partitionData.get(0).get(partitions.get(0)).offset);
    assertEquals(
        OffsetFetchResponse.INVALID_OFFSET,
        partitionData.get(0).get(partitions.get(1)).offset);

    // producer 1 now commits its transaction
    groupCoordinator.scheduleHandleTxnCompletion(
        producerIds.get(1),
        Lists.newArrayList(offsetTopicPartition).stream(),
        TransactionResult.COMMIT
    ).get();

    KeyValue<Errors, Map<TopicPartition, PartitionData>> offsetFetchResult1 =
        groupCoordinator.handleFetchOffsets(groupId, Optional.of(partitions));
    errors.add(offsetFetchResult1.getKey());
    partitionData.add(offsetFetchResult1.getValue());

    assertEquals(Errors.NONE, errors.get(1));

    assertEquals(
        offsets.get(0).offset(),
        partitionData.get(1).get(partitions.get(0)).offset);
    assertEquals(
        offsets.get(1).offset(),
        partitionData.get(1).get(partitions.get(1)).offset);
}