Java Code Examples for org.apache.pulsar.common.naming.TopicName#getPartitionIndex()

static PartitionMetadata newPartitionMetadata(TopicName topicName, Node node) {
    int pulsarPartitionIndex = topicName.getPartitionIndex();
    int kafkaPartitionIndex = pulsarPartitionIndex == -1 ? 0 : pulsarPartitionIndex;

    if (log.isDebugEnabled()) {
        log.debug("Return PartitionMetadata node: {}, topicName: {}", node, topicName);
    }

    return new PartitionMetadata(
        Errors.NONE,
        kafkaPartitionIndex,
        node,                      // leader
        Lists.newArrayList(node),  // replicas
        Lists.newArrayList(node),  // isr
        Collections.emptyList()     // offline replicas
    );
}
 

static PartitionMetadata newFailedPartitionMetadata(TopicName topicName) {
    int pulsarPartitionIndex = topicName.getPartitionIndex();
    int kafkaPartitionIndex = pulsarPartitionIndex == -1 ? 0 : pulsarPartitionIndex;

    log.warn("Failed find Broker metadata, create PartitionMetadata with NOT_LEADER_FOR_PARTITION");

    // most of this error happens when topic is in loading/unloading status,
    return new PartitionMetadata(
        Errors.NOT_LEADER_FOR_PARTITION,
        kafkaPartitionIndex,
        Node.noNode(),                      // leader
        Lists.newArrayList(Node.noNode()),  // replicas
        Lists.newArrayList(Node.noNode()),  // isr
        Collections.emptyList()             // offline replicas
    );
}
 

RawConsumerImpl(PulsarClientImpl client, ConsumerConfigurationData<byte[]> conf,
        CompletableFuture<Consumer<byte[]>> consumerFuture) {
    super(client,
        conf.getSingleTopic(),
        conf,
        client.externalExecutorProvider().getExecutor(),
        TopicName.getPartitionIndex(conf.getSingleTopic()),
        false,
        consumerFuture,
        MessageId.earliest,
        0 /* startMessageRollbackDurationInSec */,
        Schema.BYTES, null,
        true
    );
    incomingRawMessages = new GrowableArrayBlockingQueue<>();
    pendingRawReceives = new ConcurrentLinkedQueue<>();
}
 

@Override
protected void handleGetLastMessageId(CommandGetLastMessageId getLastMessageId) {
    checkArgument(state == State.Connected);

    CompletableFuture<Consumer> consumerFuture = consumers.get(getLastMessageId.getConsumerId());

    if (consumerFuture != null && consumerFuture.isDone() && !consumerFuture.isCompletedExceptionally()) {
        Consumer consumer = consumerFuture.getNow(null);
        long requestId = getLastMessageId.getRequestId();

        Topic topic = consumer.getSubscription().getTopic();
        Position position = topic.getLastPosition();
        int partitionIndex = TopicName.getPartitionIndex(topic.getName());

        getLargestBatchIndexWhenPossible(
                topic,
                (PositionImpl) position,
                partitionIndex,
                requestId,
                consumer.getSubscription().getName());

    } else {
        ctx.writeAndFlush(Commands.newError(getLastMessageId.getRequestId(), ServerError.MetadataError, "Consumer not found"));
    }
}
 

public MessageId getPositionFromSubscription(String topic, MessageId defaultPosition) {
    try {
        TopicStats topicStats = admin.topics().getStats(topic);
        if (topicStats.subscriptions.containsKey(subscriptionName)) {
            SubscriptionStats subStats = topicStats.subscriptions.get(subscriptionName);
            if (subStats.consumers.size() != 0) {
                throw new RuntimeException("Subscription been actively used by other consumers, " +
                        "in this situation, the exactly-once semantics cannot be guaranteed.");
            } else {
                PersistentTopicInternalStats.CursorStats c =
                        admin.topics().getInternalStats(topic).cursors.get(subscriptionName);
                String[] ids = c.markDeletePosition.split(":", 2);
                long ledgerId = Long.parseLong(ids[0]);
                long entryIdInMarkDelete = Long.parseLong(ids[1]);
                // we are getting the next mid from sub position, if the entryId is -1,
                // it denotes we haven't read data from the ledger before,
                // therefore no need to skip the current entry for the next position
                long entryId = entryIdInMarkDelete == -1 ? -1 : entryIdInMarkDelete + 1;
                int partitionIdx = TopicName.getPartitionIndex(topic);
                return new MessageIdImpl(ledgerId, entryId, partitionIdx);
            }
        } else {
            // create sub on topic
            log.info("Setting up subscription {} on topic {} at position {}",
                subscriptionName, topic, defaultPosition);
            admin.topics().createSubscription(topic, subscriptionName, defaultPosition);
            log.info("Subscription {} on topic {} at position {} finished",
                subscriptionName, topic, defaultPosition);
            return defaultPosition;
        }
    } catch (PulsarAdminException e) {
        throw new RuntimeException("Failed to get stats for topic " + topic, e);
    }
}
 

private void initialize(List<PulsarColumnHandle> columnHandles, PulsarSplit pulsarSplit, PulsarConnectorConfig
    pulsarConnectorConfig, ManagedLedgerFactory managedLedgerFactory, ManagedLedgerConfig managedLedgerConfig,
    PulsarConnectorMetricsTracker pulsarConnectorMetricsTracker) {
    this.columnHandles = columnHandles;
    this.pulsarSplit = pulsarSplit;
    this.partition = TopicName.getPartitionIndex(pulsarSplit.getTableName());
    this.pulsarConnectorConfig = pulsarConnectorConfig;
    this.maxBatchSize = pulsarConnectorConfig.getMaxEntryReadBatchSize();
    this.messageQueue = new SpscArrayQueue<>(pulsarConnectorConfig.getMaxSplitMessageQueueSize());
    this.entryQueue = new SpscArrayQueue<>(pulsarConnectorConfig.getMaxSplitEntryQueueSize());
    this.topicName = TopicName.get("persistent",
            NamespaceName.get(pulsarSplit.getSchemaName()),
            pulsarSplit.getTableName());
    this.metricsTracker = pulsarConnectorMetricsTracker;
    this.readOffloaded = pulsarConnectorConfig.getManagedLedgerOffloadDriver() != null;
    this.pulsarConnectorConfig = pulsarConnectorConfig;

    this.schemaHandler = PulsarSchemaHandlers
            .newPulsarSchemaHandler(this.topicName,
                    this.pulsarConnectorConfig, pulsarSplit.getSchemaInfo(), columnHandles);

    log.info("Initializing split with parameters: %s", pulsarSplit);

    try {
        this.cursor = getCursor(TopicName.get("persistent", NamespaceName.get(pulsarSplit.getSchemaName()),
            pulsarSplit.getTableName()), pulsarSplit.getStartPosition(), managedLedgerFactory, managedLedgerConfig);
    } catch (ManagedLedgerException | InterruptedException e) {
        log.error(e, "Failed to get read only cursor");
        close();
        throw new RuntimeException(e);
    }
}
 

@SuppressWarnings("unchecked")
@Override
public PulsarMessageAndMetadata<K, V> next() {

    Message<byte[]> msg = receivedMessages.poll();
    if (msg == null) {
        try {
            msg = consumer.receive();
        } catch (PulsarClientException e) {
            log.warn("Failed to receive message for {}-{}, {}", consumer.getTopic(), consumer.getSubscription(),
                    e.getMessage(), e);
            throw new RuntimeException(
                    "failed to receive message from " + consumer.getTopic() + "-" + consumer.getSubscription());
        }
    }

    int partition = TopicName.getPartitionIndex(consumer.getTopic());
    long offset = MessageIdUtils.getOffset(msg.getMessageId());
    String key = msg.getKey();
    byte[] value = msg.getValue();

    K desKey = null;
    V desValue = null;

    if (StringUtils.isNotBlank(key)) {
        if (keyDeSerializer.isPresent() && keyDeSerializer.get() instanceof StringDecoder) {
            desKey = (K) key;
        } else {
            byte[] decodedBytes = Base64.getDecoder().decode(key);
            desKey = keyDeSerializer.isPresent() ? keyDeSerializer.get().fromBytes(decodedBytes)
                    : (K) DEFAULT_DECODER.fromBytes(decodedBytes);
        }
    }

    if (value != null) {
        desValue = valueDeSerializer.isPresent() ? valueDeSerializer.get().fromBytes(msg.getData())
                : (V) DEFAULT_DECODER.fromBytes(msg.getData());
    }

    PulsarMessageAndMetadata<K, V> msgAndMetadata = new PulsarMessageAndMetadata<>(consumer.getTopic(), partition,
            null, offset, keyDeSerializer.orElse(null), valueDeSerializer.orElse(null), desKey, desValue);

    if (isAutoCommit) {
        // Commit the offset of previously dequeued messages
        consumer.acknowledgeCumulativeAsync(msg);
    }

    lastConsumedMessageId = msg.getMessageId();
    return msgAndMetadata;
}
 

@SuppressWarnings("unchecked")
@Override
public ConsumerRecords<K, V> poll(long timeoutMillis) {
    try {
        QueueItem item = receivedMessages.poll(timeoutMillis, TimeUnit.MILLISECONDS);
        if (item == null) {
            return (ConsumerRecords<K, V>) ConsumerRecords.EMPTY;
        }

        Map<TopicPartition, List<ConsumerRecord<K, V>>> records = new HashMap<>();

        int numberOfRecords = 0;

        while (item != null) {
            TopicName topicName = TopicName.get(item.consumer.getTopic());
            String topic = topicName.getPartitionedTopicName();
            int partition = topicName.isPartitioned() ? topicName.getPartitionIndex() : 0;
            Message<byte[]> msg = item.message;
            MessageIdImpl msgId = (MessageIdImpl) msg.getMessageId();
            long offset = MessageIdUtils.getOffset(msgId);

            TopicPartition tp = new TopicPartition(topic, partition);
            if (lastReceivedOffset.get(tp) == null && !unpolledPartitions.contains(tp)) {
            	log.info("When polling offsets, invalid offsets were detected. Resetting topic partition {}", tp);
            	resetOffsets(tp);
            }

            K key = getKey(topic, msg);
            if (valueSchema instanceof PulsarKafkaSchema) {
                ((PulsarKafkaSchema<V>) valueSchema).setTopic(topic);
            }
            V value = valueSchema.decode(msg.getData());

            TimestampType timestampType = TimestampType.LOG_APPEND_TIME;
            long timestamp = msg.getPublishTime();

            if (msg.getEventTime() > 0) {
                // If we have Event time, use that in preference
                timestamp = msg.getEventTime();
                timestampType = TimestampType.CREATE_TIME;
            }

            ConsumerRecord<K, V> consumerRecord = new ConsumerRecord<>(topic, partition, offset, timestamp,
                    timestampType, -1, msg.hasKey() ? msg.getKey().length() : 0, msg.getData().length, key, value);

            records.computeIfAbsent(tp, k -> new ArrayList<>()).add(consumerRecord);

            // Update last offset seen by application
            lastReceivedOffset.put(tp, offset);
            unpolledPartitions.remove(tp);

            if (++numberOfRecords >= maxRecordsInSinglePoll) {
                break;
            }

            // Check if we have an item already available
            item = receivedMessages.poll(0, TimeUnit.MILLISECONDS);
        }

        if (isAutoCommit && !records.isEmpty()) {
            // Commit the offset of previously dequeued messages
            commitAsync();
        }

        // If no interceptor is provided, interceptors list will an empty list, original ConsumerRecords will be return.
        return applyConsumerInterceptorsOnConsume(interceptors, new ConsumerRecords<>(records));
    } catch (InterruptedException e) {
        throw new RuntimeException(e);
    }
}
 

@Override
public ConsumerRecords<K, V> poll(long timeoutMillis) {
    try {
        QueueItem item = receivedMessages.poll(timeoutMillis, TimeUnit.MILLISECONDS);
        if (item == null) {
            return (ConsumerRecords<K, V>) ConsumerRecords.EMPTY;
        }

        Map<TopicPartition, List<ConsumerRecord<K, V>>> records = new HashMap<>();

        int numberOfRecords = 0;

        while (item != null) {
            TopicName topicName = TopicName.get(item.consumer.getTopic());
            String topic = topicName.getPartitionedTopicName();
            int partition = topicName.isPartitioned() ? topicName.getPartitionIndex() : 0;
            Message<byte[]> msg = item.message;
            MessageIdImpl msgId = (MessageIdImpl) msg.getMessageId();
            long offset = MessageIdUtils.getOffset(msgId);

            TopicPartition tp = new TopicPartition(topic, partition);
            if (lastReceivedOffset.get(tp) == null && !unpolledPartitions.contains(tp)) {
                log.info("When polling offsets, invalid offsets were detected. Resetting topic partition {}", tp);
                resetOffsets(tp);
            }

            K key = getKey(topic, msg);
            if (valueSchema instanceof PulsarKafkaSchema) {
                ((PulsarKafkaSchema<V>) valueSchema).setTopic(topic);
            }
            V value = valueSchema.decode(msg.getData());

            ConsumerRecord<K, V> consumerRecord = new ConsumerRecord<>(topic, partition, offset, key, value);

            records.computeIfAbsent(tp, k -> new ArrayList<>()).add(consumerRecord);

            // Update last offset seen by application
            lastReceivedOffset.put(tp, offset);
            unpolledPartitions.remove(tp);

            if (++numberOfRecords >= maxRecordsInSinglePoll) {
                break;
            }

            // Check if we have an item already available
            item = receivedMessages.poll(0, TimeUnit.MILLISECONDS);
        }

        if (isAutoCommit && !records.isEmpty()) {
            // Commit the offset of previously dequeued messages
            commitAsync();
        }

        return new ConsumerRecords<>(records);
    } catch (InterruptedException e) {
        throw new RuntimeException(e);
    }
}
 

@Override
public synchronized void addConsumer(Consumer consumer) throws BrokerServiceException {
    if (IS_FENCED_UPDATER.get(this) == TRUE) {
        log.warn("Attempting to add consumer {} on a fenced subscription", consumer);
        throw new SubscriptionFencedException("Subscription is fenced");
    }

    if (dispatcher == null || !dispatcher.isConsumerConnected()) {
        Dispatcher previousDispatcher = null;

        switch (consumer.subType()) {
        case Exclusive:
            if (dispatcher == null || dispatcher.getType() != SubType.Exclusive) {
                previousDispatcher = dispatcher;
                dispatcher = new NonPersistentDispatcherSingleActiveConsumer(SubType.Exclusive, 0, topic, this);
            }
            break;
        case Shared:
            if (dispatcher == null || dispatcher.getType() != SubType.Shared) {
                previousDispatcher = dispatcher;
                dispatcher = new NonPersistentDispatcherMultipleConsumers(topic, this);
            }
            break;
        case Failover:
            int partitionIndex = TopicName.getPartitionIndex(topicName);
            if (partitionIndex < 0) {
                // For non partition topics, assume index 0 to pick a predictable consumer
                partitionIndex = 0;
            }

            if (dispatcher == null || dispatcher.getType() != SubType.Failover) {
                previousDispatcher = dispatcher;
                dispatcher = new NonPersistentDispatcherSingleActiveConsumer(SubType.Failover, partitionIndex,
                        topic, this);
            }
            break;
        case Key_Shared:
            if (dispatcher == null || dispatcher.getType() != SubType.Key_Shared) {
                previousDispatcher = dispatcher;
                KeySharedMeta ksm = consumer.getKeySharedMeta() != null ? consumer.getKeySharedMeta() : KeySharedMeta.getDefaultInstance();

                switch (ksm.getKeySharedMode()) {
                    case STICKY:
                        dispatcher = new NonPersistentStickyKeyDispatcherMultipleConsumers(topic, this,
                                new HashRangeExclusiveStickyKeyConsumerSelector());
                        break;

                    case AUTO_SPLIT:
                    default:
                        StickyKeyConsumerSelector selector;
                        ServiceConfiguration conf = topic.getBrokerService().getPulsar().getConfiguration();
                        if (conf.isSubscriptionKeySharedUseConsistentHashing()) {
                            selector = new ConsistentHashingStickyKeyConsumerSelector(
                                    conf.getSubscriptionKeySharedConsistentHashingReplicaPoints());
                        } else {
                            selector = new HashRangeAutoSplitStickyKeyConsumerSelector();
                        }

                        dispatcher = new NonPersistentStickyKeyDispatcherMultipleConsumers(topic, this, selector);
                        break;
                }
            }
            break;
        default:
            throw new ServerMetadataException("Unsupported subscription type");
        }

        if (previousDispatcher != null) {
            previousDispatcher.close().thenRun(() -> {
                log.info("[{}][{}] Successfully closed previous dispatcher", topicName, subName);
            }).exceptionally(ex -> {
                log.error("[{}][{}] Failed to close previous dispatcher", topicName, subName, ex);
                return null;
            });
        }
    } else {
        if (consumer.subType() != dispatcher.getType()) {
            throw new SubscriptionBusyException("Subscription is of different type");
        }
    }

    dispatcher.addConsumer(consumer);
}
 

public Consumer(Subscription subscription, SubType subType, String topicName, long consumerId,
                int priorityLevel, String consumerName,
                int maxUnackedMessages, ServerCnx cnx, String appId,
                Map<String, String> metadata, boolean readCompacted, InitialPosition subscriptionInitialPosition,
                PulsarApi.KeySharedMeta keySharedMeta) throws BrokerServiceException {

    this.subscription = subscription;
    this.subType = subType;
    this.topicName = topicName;
    this.partitionIdx = TopicName.getPartitionIndex(topicName);
    this.consumerId = consumerId;
    this.priorityLevel = priorityLevel;
    this.readCompacted = readCompacted;
    this.consumerName = consumerName;
    this.maxUnackedMessages = maxUnackedMessages;
    this.subscriptionInitialPosition = subscriptionInitialPosition;
    this.keySharedMeta = keySharedMeta;
    this.cnx = cnx;
    this.msgOut = new Rate();
    this.chuckedMessageRate = new Rate();
    this.msgRedeliver = new Rate();
    this.bytesOutCounter = new LongAdder();
    this.msgOutCounter = new LongAdder();
    this.appId = appId;
    this.authenticationData = cnx.authenticationData;
    this.preciseDispatcherFlowControl = cnx.isPreciseDispatcherFlowControl();

    PERMITS_RECEIVED_WHILE_CONSUMER_BLOCKED_UPDATER.set(this, 0);
    MESSAGE_PERMITS_UPDATER.set(this, 0);
    UNACKED_MESSAGES_UPDATER.set(this, 0);
    AVG_MESSAGES_PER_ENTRY.set(this, 1000);

    this.metadata = metadata != null ? metadata : Collections.emptyMap();

    stats = new ConsumerStats();
    stats.setAddress(cnx.clientAddress().toString());
    stats.consumerName = consumerName;
    stats.setConnectedSince(DateFormatter.now());
    stats.setClientVersion(cnx.getClientVersion());
    stats.metadata = this.metadata;

    if (Subscription.isIndividualAckMode(subType)) {
        this.pendingAcks = new ConcurrentLongLongPairHashMap(256, 1);
    } else {
        // We don't need to keep track of pending acks if the subscription is not shared
        this.pendingAcks = null;
    }
}
 

@Override
public synchronized void addConsumer(Consumer consumer) throws BrokerServiceException {
    cursor.updateLastActive();
    if (IS_FENCED_UPDATER.get(this) == TRUE) {
        log.warn("Attempting to add consumer {} on a fenced subscription", consumer);
        throw new SubscriptionFencedException("Subscription is fenced");
    }

    if (dispatcher == null || !dispatcher.isConsumerConnected()) {
        Dispatcher previousDispatcher = null;

        switch (consumer.subType()) {
        case Exclusive:
            if (dispatcher == null || dispatcher.getType() != SubType.Exclusive) {
                previousDispatcher = dispatcher;
                dispatcher = new PersistentDispatcherSingleActiveConsumer(cursor, SubType.Exclusive, 0, topic, this);
            }
            break;
        case Shared:
            if (dispatcher == null || dispatcher.getType() != SubType.Shared) {
                previousDispatcher = dispatcher;
                dispatcher = new PersistentDispatcherMultipleConsumers(topic, cursor, this);
            }
            break;
        case Failover:
            int partitionIndex = TopicName.getPartitionIndex(topicName);
            if (partitionIndex < 0) {
                // For non partition topics, use a negative index so dispatcher won't sort consumers before picking
                // an active consumer for the topic.
                partitionIndex = -1;
            }

            if (dispatcher == null || dispatcher.getType() != SubType.Failover) {
                previousDispatcher = dispatcher;
                dispatcher = new PersistentDispatcherSingleActiveConsumer(cursor, SubType.Failover, partitionIndex,
                        topic, this);
            }
            break;
        case Key_Shared:
            if (dispatcher == null || dispatcher.getType() != SubType.Key_Shared) {
                previousDispatcher = dispatcher;
                KeySharedMeta ksm = consumer.getKeySharedMeta() != null ? consumer.getKeySharedMeta()
                        : KeySharedMeta.getDefaultInstance();
                dispatcher = new PersistentStickyKeyDispatcherMultipleConsumers(topic, cursor, this,
                        topic.getBrokerService().getPulsar().getConfiguration(), ksm);
            }
            break;
        default:
            throw new ServerMetadataException("Unsupported subscription type");
        }

        if (previousDispatcher != null) {
            previousDispatcher.close().thenRun(() -> {
                log.info("[{}][{}] Successfully closed previous dispatcher", topicName, subName);
            }).exceptionally(ex -> {
                log.error("[{}][{}] Failed to close previous dispatcher", topicName, subName, ex);
                return null;
            });
        }
    } else {
        if (consumer.subType() != dispatcher.getType()) {
            throw new SubscriptionBusyException("Subscription is of different type");
        }
    }

    dispatcher.addConsumer(consumer);
}