Java Code Examples for org.apache.pulsar.common.naming.TopicName#getNamespace()

The following examples show how to use org.apache.pulsar.common.naming.TopicName#getNamespace() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: BacklogQuotaManager.java    From pulsar with Apache License 2.0 6 votes vote down vote up 
/**
 * Handle exceeded backlog by using policies set in the zookeeper for given topic
 *
 * @param persistentTopic
 *            Topic on which backlog has been exceeded
 */
public void handleExceededBacklogQuota(PersistentTopic persistentTopic) {
    TopicName topicName = TopicName.get(persistentTopic.getName());
    String namespace = topicName.getNamespace();
    String policyPath = AdminResource.path(POLICIES, namespace);

    BacklogQuota quota = getBacklogQuota(namespace, policyPath);
    log.info("Backlog quota exceeded for topic [{}]. Applying [{}] policy", persistentTopic.getName(),
            quota.getPolicy());
    switch (quota.getPolicy()) {
    case consumer_backlog_eviction:
        dropBacklog(persistentTopic, quota);
        break;
    case producer_exception:
    case producer_request_hold:
        disconnectProducers(persistentTopic);
        break;
    default:
        break;
    }
}
Example 2
Source File: TestPulsarSplitManager.java    From pulsar with Apache License 2.0 6 votes vote down vote up 
@Test(dataProvider = "rewriteNamespaceDelimiter", singleThreaded = true)
public void testGetSplitNonSchema(String delimiter) throws Exception {
    updateRewriteNamespaceDelimiterIfNeeded(delimiter);
    TopicName topicName = NON_SCHEMA_TOPIC;
    setup();
    log.info("!----- topic: %s -----!", topicName);
    PulsarTableHandle pulsarTableHandle = new PulsarTableHandle(pulsarConnectorId.toString(),
        topicName.getNamespace(),
        topicName.getLocalName(),
        topicName.getLocalName());

    Map<ColumnHandle, Domain> domainMap = new HashMap<>();
    TupleDomain<ColumnHandle> tupleDomain = TupleDomain.withColumnDomains(domainMap);

    PulsarTableLayoutHandle pulsarTableLayoutHandle = new PulsarTableLayoutHandle(pulsarTableHandle, tupleDomain);
    ConnectorSplitSource connectorSplitSource = this.pulsarSplitManager.getSplits(
        mock(ConnectorTransactionHandle.class), mock(ConnectorSession.class),
        pulsarTableLayoutHandle, null);
    assertNotNull(connectorSplitSource);
}
Example 3
Source File: PersistentTopic.java    From pulsar with Apache License 2.0 5 votes vote down vote up 
/**
 *
 * @return Backlog quota for topic
 */
@Override
public BacklogQuota getBacklogQuota() {
    TopicName topicName = TopicName.get(this.getName());
    String namespace = topicName.getNamespace();
    String policyPath = AdminResource.path(POLICIES, namespace);

    BacklogQuota backlogQuota = brokerService.getBacklogQuotaManager().getBacklogQuota(namespace, policyPath);
    return backlogQuota;
}
Example 4
Source File: BookieClientStatsGenerator.java    From pulsar with Apache License 2.0 5 votes vote down vote up 
private void put(TopicName topicName, PendingBookieOpsStats bookieOpsStats) {
    String namespace = topicName.getNamespace();
    if (!nsBookieClientStatsMap.containsKey(namespace)) {
        Map<String, PendingBookieOpsStats> destBookieClientStatsMap = Maps.newTreeMap();
        destBookieClientStatsMap.put(topicName.toString(), bookieOpsStats);
        nsBookieClientStatsMap.put(namespace, destBookieClientStatsMap);
    } else {
        nsBookieClientStatsMap.get(namespace).put(topicName.toString(), bookieOpsStats);
    }

}
Example 5
Source File: ReplicatorTestBase.java    From pulsar with Apache License 2.0 5 votes vote down vote up 
MessageProducer(URL url, final TopicName dest) throws Exception {
    this.url = url;
    this.namespace = dest.getNamespace();
    this.topicName = dest.toString();
    client = PulsarClient.builder().serviceUrl(url.toString()).statsInterval(0, TimeUnit.SECONDS).build();
    producer = client.newProducer()
        .topic(topicName)
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.SinglePartition)
        .create();

}
Example 6
Source File: ReplicatorTestBase.java    From pulsar with Apache License 2.0 5 votes vote down vote up 
MessageProducer(URL url, final TopicName dest, boolean batch) throws Exception {
    this.url = url;
    this.namespace = dest.getNamespace();
    this.topicName = dest.toString();
    client = PulsarClient.builder().serviceUrl(url.toString()).statsInterval(0, TimeUnit.SECONDS).build();
    ProducerBuilder<byte[]> producerBuilder = client.newProducer()
        .topic(topicName)
        .enableBatching(batch)
        .batchingMaxPublishDelay(1, TimeUnit.SECONDS)
        .batchingMaxMessages(5);
    producer = producerBuilder.create();

}
Example 7
Source File: ReplicatorTestBase.java    From pulsar with Apache License 2.0 5 votes vote down vote up 
MessageConsumer(URL url, final TopicName dest, String subId) throws Exception {
    this.url = url;
    this.namespace = dest.getNamespace();
    this.topicName = dest.toString();

    client = PulsarClient.builder().serviceUrl(url.toString()).statsInterval(0, TimeUnit.SECONDS).build();

    try {
        consumer = client.newConsumer().topic(topicName).subscriptionName(subId).subscribe();
    } catch (Exception e) {
        client.close();
        throw e;
    }
}
Example 8
Source File: ConsumerBuilderImpl.java    From pulsar with Apache License 2.0 5 votes vote down vote up 
@Override
public CompletableFuture<Consumer<T>> subscribeAsync() {
    if (conf.getTopicNames().isEmpty() && conf.getTopicsPattern() == null) {
        return FutureUtil
                .failedFuture(new InvalidConfigurationException("Topic name must be set on the consumer builder"));
    }

    if (StringUtils.isBlank(conf.getSubscriptionName())) {
        return FutureUtil.failedFuture(
                new InvalidConfigurationException("Subscription name must be set on the consumer builder"));
    }

    if (conf.getKeySharedPolicy() != null && conf.getSubscriptionType() != SubscriptionType.Key_Shared) {
        return FutureUtil.failedFuture(
                new InvalidConfigurationException("KeySharedPolicy must set with KeyShared subscription"));
    }
    if(conf.isRetryEnable() && conf.getTopicNames().size() > 0 ) {
        TopicName topicFirst = TopicName.get(conf.getTopicNames().iterator().next());
        String retryLetterTopic = topicFirst.getNamespace() + "/" + conf.getSubscriptionName() + RetryMessageUtil.RETRY_GROUP_TOPIC_SUFFIX;
        String deadLetterTopic = topicFirst.getNamespace() + "/" + conf.getSubscriptionName() + RetryMessageUtil.DLQ_GROUP_TOPIC_SUFFIX;
        if(conf.getDeadLetterPolicy() == null) {
            conf.setDeadLetterPolicy(DeadLetterPolicy.builder()
                                    .maxRedeliverCount(RetryMessageUtil.MAX_RECONSUMETIMES)
                                    .retryLetterTopic(retryLetterTopic)
                                    .deadLetterTopic(deadLetterTopic)
                                    .build());
        } else {
            if (StringUtils.isBlank(conf.getDeadLetterPolicy().getRetryLetterTopic())) {
                conf.getDeadLetterPolicy().setRetryLetterTopic(retryLetterTopic);
            }
            if (StringUtils.isBlank(conf.getDeadLetterPolicy().getDeadLetterTopic())) {
                conf.getDeadLetterPolicy().setDeadLetterTopic(deadLetterTopic);
            }
        }
        conf.getTopicNames().add(conf.getDeadLetterPolicy().getRetryLetterTopic());
    }
    return interceptorList == null || interceptorList.size() == 0 ?
            client.subscribeAsync(conf, schema, null) :
            client.subscribeAsync(conf, schema, new ConsumerInterceptors<>(interceptorList));
}
Example 9
Source File: TestPulsarSplitManager.java    From pulsar with Apache License 2.0 4 votes vote down vote up 
@Test(dataProvider = "rewriteNamespaceDelimiter", singleThreaded = true)
public void testPartitionedTopic(String delimiter) throws Exception {
    updateRewriteNamespaceDelimiterIfNeeded(delimiter);
    for (TopicName topicName : partitionedTopicNames) {
        setup();
        log.info("!----- topic: %s -----!", topicName);
        PulsarTableHandle pulsarTableHandle = new PulsarTableHandle(pulsarConnectorId.toString(),
                topicName.getNamespace(),
                topicName.getLocalName(),
                topicName.getLocalName());
        PulsarTableLayoutHandle pulsarTableLayoutHandle = new PulsarTableLayoutHandle(pulsarTableHandle, TupleDomain.all());

        final ResultCaptor<Collection<PulsarSplit>> resultCaptor = new ResultCaptor<>();
        doAnswer(resultCaptor).when(this.pulsarSplitManager).getSplitsPartitionedTopic(anyInt(), any(), any(), any(), any(), any());

        this.pulsarSplitManager.getSplits(mock(ConnectorTransactionHandle.class), mock(ConnectorSession.class),
                pulsarTableLayoutHandle, null);

        verify(this.pulsarSplitManager, times(1))
                .getSplitsPartitionedTopic(anyInt(), any(), any(), any(), any(), any());

        int partitions = partitionedTopicsToPartitions.get(topicName.toString());

        for (int i = 0; i < partitions; i++) {
            List<PulsarSplit> splits = getSplitsForPartition(topicName.getPartition(i), resultCaptor.getResult());
            int totalSize = 0;
            for (PulsarSplit pulsarSplit : splits) {
                assertEquals(pulsarSplit.getConnectorId(), pulsarConnectorId.toString());
                assertEquals(pulsarSplit.getSchemaName(), topicName.getNamespace());
                assertEquals(pulsarSplit.getTableName(), topicName.getPartition(i).getLocalName());
                assertEquals(pulsarSplit.getSchema(),
                        new String(topicsToSchemas.get(topicName.getSchemaName()).getSchema()));
                assertEquals(pulsarSplit.getSchemaType(), topicsToSchemas.get(topicName.getSchemaName()).getType());
                assertEquals(pulsarSplit.getStartPositionEntryId(), totalSize);
                assertEquals(pulsarSplit.getStartPositionLedgerId(), 0);
                assertEquals(pulsarSplit.getStartPosition(), PositionImpl.get(0, totalSize));
                assertEquals(pulsarSplit.getEndPositionLedgerId(), 0);
                assertEquals(pulsarSplit.getEndPositionEntryId(), totalSize + pulsarSplit.getSplitSize());
                assertEquals(pulsarSplit.getEndPosition(), PositionImpl.get(0, totalSize + pulsarSplit.getSplitSize()));

                totalSize += pulsarSplit.getSplitSize();
            }

            assertEquals(totalSize, topicsToNumEntries.get(topicName.getSchemaName()).intValue());
        }

        cleanup();
    }
}
Example 10
Source File: TestPulsarSplitManager.java    From pulsar with Apache License 2.0 4 votes vote down vote up 
@Test(dataProvider = "rewriteNamespaceDelimiter", singleThreaded = true)
public void testPublishTimePredicatePushdown(String delimiter) throws Exception {
    updateRewriteNamespaceDelimiterIfNeeded(delimiter);
    TopicName topicName = TOPIC_1;

    setup();
    log.info("!----- topic: %s -----!", topicName);
    PulsarTableHandle pulsarTableHandle = new PulsarTableHandle(pulsarConnectorId.toString(),
            topicName.getNamespace(),
            topicName.getLocalName(),
            topicName.getLocalName());


    Map<ColumnHandle, Domain> domainMap = new HashMap<>();
    Domain domain = Domain.create(ValueSet.ofRanges(Range.range(TIMESTAMP, currentTimeMs + 1L, true,
            currentTimeMs + 50L, true)), false);
    domainMap.put(PulsarInternalColumn.PUBLISH_TIME.getColumnHandle(pulsarConnectorId.toString(), false), domain);
    TupleDomain<ColumnHandle> tupleDomain = TupleDomain.withColumnDomains(domainMap);

    PulsarTableLayoutHandle pulsarTableLayoutHandle = new PulsarTableLayoutHandle(pulsarTableHandle, tupleDomain);

    final ResultCaptor<Collection<PulsarSplit>> resultCaptor = new ResultCaptor<>();
    doAnswer(resultCaptor).when(this.pulsarSplitManager)
            .getSplitsNonPartitionedTopic(anyInt(), any(), any(), any(), any(), any());

    ConnectorSplitSource connectorSplitSource = this.pulsarSplitManager.getSplits(
            mock(ConnectorTransactionHandle.class), mock(ConnectorSession.class),
            pulsarTableLayoutHandle, null);


    verify(this.pulsarSplitManager, times(1))
            .getSplitsNonPartitionedTopic(anyInt(), any(), any(), any(), any(), any());

    int totalSize = 0;
    int initalStart = 1;
    for (PulsarSplit pulsarSplit : resultCaptor.getResult()) {
        assertEquals(pulsarSplit.getConnectorId(), pulsarConnectorId.toString());
        assertEquals(pulsarSplit.getSchemaName(), topicName.getNamespace());
        assertEquals(pulsarSplit.getTableName(), topicName.getLocalName());
        assertEquals(pulsarSplit.getSchema(),
                new String(topicsToSchemas.get(topicName.getSchemaName()).getSchema()));
        assertEquals(pulsarSplit.getSchemaType(), topicsToSchemas.get(topicName.getSchemaName()).getType());
        assertEquals(pulsarSplit.getStartPositionEntryId(), initalStart);
        assertEquals(pulsarSplit.getStartPositionLedgerId(), 0);
        assertEquals(pulsarSplit.getStartPosition(), PositionImpl.get(0, initalStart));
        assertEquals(pulsarSplit.getEndPositionLedgerId(), 0);
        assertEquals(pulsarSplit.getEndPositionEntryId(), initalStart + pulsarSplit.getSplitSize());
        assertEquals(pulsarSplit.getEndPosition(), PositionImpl.get(0, initalStart + pulsarSplit
                .getSplitSize()));

        initalStart += pulsarSplit.getSplitSize();
        totalSize += pulsarSplit.getSplitSize();
    }
    assertEquals(totalSize, 49);

}
Example 11
Source File: TestPulsarSplitManager.java    From pulsar with Apache License 2.0 4 votes vote down vote up 
@Test(dataProvider = "rewriteNamespaceDelimiter", singleThreaded = true)
public void testPublishTimePredicatePushdownPartitionedTopic(String delimiter) throws Exception {
    updateRewriteNamespaceDelimiterIfNeeded(delimiter);
    TopicName topicName = PARTITIONED_TOPIC_1;

    setup();
    log.info("!----- topic: %s -----!", topicName);
    PulsarTableHandle pulsarTableHandle = new PulsarTableHandle(pulsarConnectorId.toString(),
            topicName.getNamespace(),
            topicName.getLocalName(),
            topicName.getLocalName());


    Map<ColumnHandle, Domain> domainMap = new HashMap<>();
    Domain domain = Domain.create(ValueSet.ofRanges(Range.range(TIMESTAMP, currentTimeMs + 1L, true,
            currentTimeMs + 50L, true)), false);
    domainMap.put(PulsarInternalColumn.PUBLISH_TIME.getColumnHandle(pulsarConnectorId.toString(), false), domain);
    TupleDomain<ColumnHandle> tupleDomain = TupleDomain.withColumnDomains(domainMap);

    PulsarTableLayoutHandle pulsarTableLayoutHandle = new PulsarTableLayoutHandle(pulsarTableHandle, tupleDomain);

    final ResultCaptor<Collection<PulsarSplit>> resultCaptor = new ResultCaptor<>();
    doAnswer(resultCaptor).when(this.pulsarSplitManager)
            .getSplitsPartitionedTopic(anyInt(), any(), any(), any(), any(), any());

    ConnectorSplitSource connectorSplitSource = this.pulsarSplitManager.getSplits(
            mock(ConnectorTransactionHandle.class), mock(ConnectorSession.class),
            pulsarTableLayoutHandle, null);


    verify(this.pulsarSplitManager, times(1))
            .getSplitsPartitionedTopic(anyInt(), any(), any(), any(), any(), any());


    int partitions = partitionedTopicsToPartitions.get(topicName.toString());
    for (int i = 0; i < partitions; i++) {
        List<PulsarSplit> splits = getSplitsForPartition(topicName.getPartition(i), resultCaptor.getResult());
        int totalSize = 0;
        int initialStart = 1;
        for (PulsarSplit pulsarSplit : splits) {
            assertEquals(pulsarSplit.getConnectorId(), pulsarConnectorId.toString());
            assertEquals(pulsarSplit.getSchemaName(), topicName.getNamespace());
            assertEquals(pulsarSplit.getTableName(), topicName.getPartition(i).getLocalName());
            assertEquals(pulsarSplit.getSchema(),
                    new String(topicsToSchemas.get(topicName.getSchemaName()).getSchema()));
            assertEquals(pulsarSplit.getSchemaType(), topicsToSchemas.get(topicName.getSchemaName()).getType());
            assertEquals(pulsarSplit.getStartPositionEntryId(), initialStart);
            assertEquals(pulsarSplit.getStartPositionLedgerId(), 0);
            assertEquals(pulsarSplit.getStartPosition(), PositionImpl.get(0, initialStart));
            assertEquals(pulsarSplit.getEndPositionLedgerId(), 0);
            assertEquals(pulsarSplit.getEndPositionEntryId(), initialStart + pulsarSplit.getSplitSize());
            assertEquals(pulsarSplit.getEndPosition(), PositionImpl.get(0, initialStart + pulsarSplit.getSplitSize()));

            initialStart += pulsarSplit.getSplitSize();
            totalSize += pulsarSplit.getSplitSize();
        }

        assertEquals(totalSize, 49);
    }
}
Example 12
Source File: TestPulsarMetadata.java    From pulsar with Apache License 2.0 4 votes vote down vote up 
@Test(dataProvider = "rewriteNamespaceDelimiter", singleThreaded = true)
public void testGetTableMetadata(String delimiter) {
    updateRewriteNamespaceDelimiterIfNeeded(delimiter);
    List<TopicName> allTopics = new LinkedList<>();
    allTopics.addAll(topicNames.stream().filter(topicName -> !topicName.equals(NON_SCHEMA_TOPIC)).collect(Collectors.toList()));
    allTopics.addAll(partitionedTopicNames);

    for (TopicName topic : allTopics) {
        PulsarTableHandle pulsarTableHandle = new PulsarTableHandle(
                topic.toString(),
                topic.getNamespace(),
                topic.getLocalName(),
                topic.getLocalName()
        );

        ConnectorTableMetadata tableMetadata = this.pulsarMetadata.getTableMetadata(mock(ConnectorSession.class),
                pulsarTableHandle);

        assertEquals(tableMetadata.getTable().getSchemaName(), topic.getNamespace());
        assertEquals(tableMetadata.getTable().getTableName(), topic.getLocalName());
        assertEquals(tableMetadata.getColumns().size(),
                fooColumnHandles.size());

        List<String> fieldNames = new LinkedList<>(fooFieldNames.keySet());

        for (PulsarInternalColumn internalField : PulsarInternalColumn.getInternalFields()) {
            fieldNames.add(internalField.getName());
        }

        for (ColumnMetadata column : tableMetadata.getColumns()) {
            if (PulsarInternalColumn.getInternalFieldsMap().containsKey(column.getName())) {
                assertEquals(column.getComment(),
                        PulsarInternalColumn.getInternalFieldsMap()
                                .get(column.getName()).getColumnMetadata(true).getComment());
            }

            fieldNames.remove(column.getName());
        }

        assertTrue(fieldNames.isEmpty());
    }
}