org.apache.pulsar.common.policies.data.TopicStats Java Examples

@Test
public void createReplicatedSubscription() throws Exception {
    String topic = "createReplicatedSubscription-" + System.nanoTime();

    @Cleanup
    Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING)
            .topic(topic)
            .subscriptionName("sub1")
            .replicateSubscriptionState(true)
            .subscribe();

    TopicStats stats = admin.topics().getStats(topic);
    assertTrue(stats.subscriptions.get("sub1").isReplicated);

    admin.topics().unload(topic);

    // Check that subscription is still marked replicated after reloading
    stats = admin.topics().getStats(topic);
    assertTrue(stats.subscriptions.get("sub1").isReplicated);
}
 

@Test
public void testTenantNameWithUnderscore() throws Exception {
    TenantInfo tenantInfo = new TenantInfo(Sets.newHashSet("role1", "role2"), Sets.newHashSet("test"));
    admin.tenants().createTenant("prop_xyz", tenantInfo);

    admin.namespaces().createNamespace("prop_xyz/my-namespace", Sets.newHashSet("test"));

    String topic = "persistent://prop_xyz/use/my-namespace/my-topic";

    Producer<byte[]> producer = pulsarClient.newProducer().topic(topic)
            .create();

    TopicStats stats = admin.topics().getStats(topic);
    assertEquals(stats.publishers.size(), 1);
    producer.close();
}
 

@Override
public CompletableFuture<TopicStats> getStatsAsync(String topic, boolean getPreciseBacklog) {
    TopicName tn = validateTopic(topic);
    WebTarget path = topicPath(tn, "stats").queryParam("getPreciseBacklog", getPreciseBacklog);
    final CompletableFuture<TopicStats> future = new CompletableFuture<>();
    asyncGetRequest(path,
            new InvocationCallback<TopicStats>() {

                @Override
                public void completed(TopicStats response) {
                    future.complete(response);
                }

                @Override
                public void failed(Throwable throwable) {
                    future.completeExceptionally(getApiException(throwable.getCause()));
                }
            });
    return future;
}
 

@GET
@Path("{tenant}/{namespace}/{topic}/stats")
@ApiOperation(value = "Get the stats for the topic.")
@ApiResponses(value = {
        @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"),
        @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant"),
        @ApiResponse(code = 403, message = "Don't have admin permission"),
        @ApiResponse(code = 404, message = "Topic does not exist"),
        @ApiResponse(code = 412, message = "Topic name is not valid"),
        @ApiResponse(code = 500, message = "Internal server error"),
        @ApiResponse(code = 503, message = "Failed to validate global cluster configuration") })
public TopicStats getStats(
        @ApiParam(value = "Specify the tenant", required = true)
        @PathParam("tenant") String tenant,
        @ApiParam(value = "Specify the namespace", required = true)
        @PathParam("namespace") String namespace,
        @ApiParam(value = "Specify topic name", required = true)
        @PathParam("topic") @Encoded String encodedTopic,
        @ApiParam(value = "Is authentication required to perform this operation")
        @QueryParam("authoritative") @DefaultValue("false") boolean authoritative,
        @ApiParam(value = "Is return precise backlog or imprecise backlog")
        @QueryParam("getPreciseBacklog") @DefaultValue("false") boolean getPreciseBacklog) {
    validateTopicName(tenant, namespace, encodedTopic);
    return internalGetStats(authoritative, getPreciseBacklog);
}
 

@Test
public void testSharedProducer() throws Exception {
    TopicStats topicStats = admin.topics().getStats(topic);
    Assert.assertEquals(topicStats.publishers.size(), 1);
    PulsarBolt otherBolt = new PulsarBolt(pulsarBoltConf, PulsarClient.builder());
    MockOutputCollector otherMockCollector = new MockOutputCollector();
    OutputCollector collector = new OutputCollector(otherMockCollector);
    TopologyContext context = mock(TopologyContext.class);
    when(context.getThisComponentId()).thenReturn("test-bolt-" + methodName);
    when(context.getThisTaskId()).thenReturn(1);
    otherBolt.prepare(Maps.newHashMap(), context, collector);

    topicStats = admin.topics().getStats(topic);
    Assert.assertEquals(topicStats.publishers.size(), 1);

    otherBolt.close();

    topicStats = admin.topics().getStats(topic);
    Assert.assertEquals(topicStats.publishers.size(), 1);
}
 

@Test
public void testNoSharedConsumer() throws Exception {
    TopicStats topicStats = admin.topics().getStats(topic);
    assertEquals(topicStats.subscriptions.get(subscriptionName).consumers.size(), 1);
    pulsarSpoutConf.setSharedConsumerEnabled(false);
    PulsarSpout otherSpout = new PulsarSpout(pulsarSpoutConf, PulsarClient.builder());
    MockSpoutOutputCollector otherMockCollector = new MockSpoutOutputCollector();
    SpoutOutputCollector collector = new SpoutOutputCollector(otherMockCollector);
    TopologyContext context = mock(TopologyContext.class);
    when(context.getThisComponentId()).thenReturn("test-spout-" + methodName);
    when(context.getThisTaskId()).thenReturn(1);
    otherSpout.open(Maps.newHashMap(), context, collector);

    topicStats = admin.topics().getStats(topic);
    assertEquals(topicStats.subscriptions.get(subscriptionName).consumers.size(), 2);

    otherSpout.close();

    topicStats = admin.topics().getStats(topic);
    assertEquals(topicStats.subscriptions.get(subscriptionName).consumers.size(), 1);
}
 

@Test
public void testSharedConsumer() throws Exception {
    TopicStats topicStats = admin.topics().getStats(topic);
    assertEquals(topicStats.subscriptions.get(subscriptionName).consumers.size(), 1);
    PulsarSpout otherSpout = new PulsarSpout(pulsarSpoutConf, PulsarClient.builder());
    MockSpoutOutputCollector otherMockCollector = new MockSpoutOutputCollector();
    SpoutOutputCollector collector = new SpoutOutputCollector(otherMockCollector);
    TopologyContext context = mock(TopologyContext.class);
    when(context.getThisComponentId()).thenReturn("test-spout-" + methodName);
    when(context.getThisTaskId()).thenReturn(1);
    otherSpout.open(Maps.newHashMap(), context, collector);

    topicStats = admin.topics().getStats(topic);
    assertEquals(topicStats.subscriptions.get(subscriptionName).consumers.size(), 1);

    otherSpout.close();

    topicStats = admin.topics().getStats(topic);
    assertEquals(topicStats.subscriptions.get(subscriptionName).consumers.size(), 1);
}
 

@Test
public void upgradeToReplicatedSubscription() throws Exception {
    String topic = "upgradeToReplicatedSubscription-" + System.nanoTime();

    Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING)
            .topic(topic)
            .subscriptionName("sub")
            .replicateSubscriptionState(false)
            .subscribe();

    TopicStats stats = admin.topics().getStats(topic);
    assertFalse(stats.subscriptions.get("sub").isReplicated);
    consumer.close();

    consumer = pulsarClient.newConsumer(Schema.STRING)
            .topic(topic)
            .subscriptionName("sub")
            .replicateSubscriptionState(true)
            .subscribe();

    stats = admin.topics().getStats(topic);
    assertTrue(stats.subscriptions.get("sub").isReplicated);
    consumer.close();
}
 

@Test
public void upgradeToReplicatedSubscriptionAfterRestart() throws Exception {
    String topic = "upgradeToReplicatedSubscriptionAfterRestart-" + System.nanoTime();

    Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING)
            .topic(topic)
            .subscriptionName("sub")
            .replicateSubscriptionState(false)
            .subscribe();

    TopicStats stats = admin.topics().getStats(topic);
    assertFalse(stats.subscriptions.get("sub").isReplicated);
    consumer.close();

    admin.topics().unload(topic);

    consumer = pulsarClient.newConsumer(Schema.STRING)
            .topic(topic)
            .subscriptionName("sub")
            .replicateSubscriptionState(true)
            .subscribe();

    stats = admin.topics().getStats(topic);
    assertTrue(stats.subscriptions.get("sub").isReplicated);
    consumer.close();
}
 

public WorkerInfo getLeader() {
    TopicStats topicStats = null;
    try {
        topicStats = this.pulsarAdmin.topics().getStats(this.workerConfig.getClusterCoordinationTopic());
    } catch (PulsarAdminException e) {
        log.error("Failed to get status of coordinate topic {}",
                this.workerConfig.getClusterCoordinationTopic(), e);
        throw new RuntimeException(e);
    }

    String activeConsumerName = topicStats.subscriptions.get(COORDINATION_TOPIC_SUBSCRIPTION).activeConsumerName;
    WorkerInfo leader = null;
    for (ConsumerStats consumerStats : topicStats.subscriptions
            .get(COORDINATION_TOPIC_SUBSCRIPTION).consumers) {
        if (consumerStats.consumerName.equals(activeConsumerName)) {
            leader = WorkerInfo.parseFrom(consumerStats.metadata.get(WORKER_IDENTIFIER));
        }
    }
    if (leader == null) {
        log.warn("Failed to determine leader in functions cluster");
    }
    return leader;
}
 

@Test
public void testTopicStats() throws Exception {
    String topicName = "persistent://my-property/my-ns/testTopicStats";

    Reader<byte[]> reader1 = pulsarClient.newReader().topic(topicName).startMessageId(MessageId.earliest).create();

    Reader<byte[]> reader2 = pulsarClient.newReader().topic(topicName).startMessageId(MessageId.earliest).create();

    TopicStats stats = admin.topics().getStats(topicName);
    assertEquals(stats.subscriptions.size(), 2);

    reader1.close();
    stats = admin.topics().getStats(topicName);
    assertEquals(stats.subscriptions.size(), 1);

    reader2.close();

    stats = admin.topics().getStats(topicName);
    assertEquals(stats.subscriptions.size(), 0);
}
 

public List<WorkerInfo> getCurrentMembership() {

        List<WorkerInfo> workerIds = new LinkedList<>();
        TopicStats topicStats = null;
        try {
            topicStats = this.pulsarAdmin.topics().getStats(this.workerConfig.getClusterCoordinationTopic());
        } catch (PulsarAdminException e) {
            log.error("Failed to get status of coordinate topic {}",
                    this.workerConfig.getClusterCoordinationTopic(), e);
            throw new RuntimeException(e);
        }

        for (ConsumerStats consumerStats : topicStats.subscriptions
                .get(COORDINATION_TOPIC_SUBSCRIPTION).consumers) {
            WorkerInfo workerInfo = WorkerInfo.parseFrom(consumerStats.metadata.get(WORKER_IDENTIFIER));
            workerIds.add(workerInfo);
        }
        return workerIds;
    }
 

@Test
public void testTenantNameWithUnderscore() throws Exception {
    TenantInfo tenantInfo = new TenantInfo(Sets.newHashSet("role1", "role2"), Sets.newHashSet("use"));
    admin.tenants().createTenant("prop_xyz", tenantInfo);

    admin.namespaces().createNamespace("prop_xyz/use/my-namespace");

    String topic = "persistent://prop_xyz/use/my-namespace/my-topic";

    Producer<byte[]> producer = pulsarClient.newProducer()
        .topic(topic)
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.SinglePartition)
        .create();

    TopicStats stats = admin.topics().getStats(topic);
    assertEquals(stats.publishers.size(), 1);
    producer.close();
}
 

@Test
public void testBacklogSizeShouldBeZeroWhenConsumerAckedAllMessages() throws Exception {
    final String topic = "persistent://prop-xyz/ns1/testBacklogSizeShouldBeZeroWhenConsumerAckedAllMessages";
    Consumer<byte[]> consumer = pulsarClient.newConsumer()
            .topic(topic)
            .subscriptionName("sub-1")
            .subscribe();
    Producer<byte[]> producer = pulsarClient.newProducer()
            .topic(topic)
            .create();

    final int messages = 33;
    for (int i = 0; i < messages; i++) {
        producer.send(new byte[1024 * i * 5]);
    }

    for (int i = 0; i < messages; i++) {
        consumer.acknowledgeCumulative(consumer.receive());
    }

    // Wait ack send
    Thread.sleep(1000);

    TopicStats topicStats = admin.topics().getStats(topic);
    assertEquals(topicStats.backlogSize, 0);
}
 

@Test
public void testAheadProducerOnHold() throws Exception {
    assertEquals(admin.namespaces().getBacklogQuotaMap("prop/quotahold"),
            ConfigHelper.backlogQuotaMap(config));
    admin.namespaces().setBacklogQuota("prop/quotahold",
            new BacklogQuota(10 * 1024, BacklogQuota.RetentionPolicy.producer_request_hold));
    final PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString())
            .statsInterval(0, TimeUnit.SECONDS).build();
    final String topic1 = "persistent://prop/quotahold/hold";
    final String subName1 = "c1hold";
    final int numMsgs = 10;

    Consumer<byte[]> consumer = client.newConsumer().topic(topic1).subscriptionName(subName1).subscribe();

    byte[] content = new byte[1024];
    Producer<byte[]> producer = client.newProducer().topic(topic1).sendTimeout(2, TimeUnit.SECONDS).create();
    for (int i = 0; i <= numMsgs; i++) {
        try {
            producer.send(content);
            LOG.info("sent [{}]", i);
        } catch (PulsarClientException.TimeoutException cte) {
            // producer close may cause a timeout on send
            LOG.info("timeout on [{}]", i);
        }
    }

    for (int i = 0; i < numMsgs; i++) {
        consumer.receive();
        LOG.info("received [{}]", i);
    }

    Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000);
    rolloverStats();
    TopicStats stats = admin.topics().getStats(topic1);
    assertEquals(stats.publishers.size(), 0,
            "Number of producers on topic " + topic1 + " are [" + stats.publishers.size() + "]");
    client.close();
}
 

@Test
public void testPreciseBacklogForPartitionedTopic() throws PulsarClientException, PulsarAdminException, InterruptedException {
    final String topic = "persistent://prop-xyz/ns1/precise-back-log-for-partitioned-topic";
    admin.topics().createPartitionedTopic(topic, 2);
    final String subName = "sub-name";

    @Cleanup
    PulsarClient client = PulsarClient.builder().serviceUrl(pulsar.getWebServiceAddress()).build();

    @Cleanup
    Consumer<byte[]> consumer = client.newConsumer()
        .topic(topic)
        .subscriptionName(subName)
        .subscribe();

    @Cleanup
    Producer<byte[]> producer = client.newProducer()
        .topic(topic)
        .enableBatching(false)
        .create();

    producer.send("message-1".getBytes(StandardCharsets.UTF_8));
    Message<byte[]> message = consumer.receive();
    assertNotNull(message);

    // Mock the entries added count. Default is disable the precise backlog, so the backlog is entries added count - consumed count
    // Since message have not acked, so the backlog is 10
    for (int i = 0; i < 2; i++) {
        PersistentSubscription subscription = (PersistentSubscription)pulsar.getBrokerService().getTopicReference(topic + "-partition-" + i).get().getSubscription(subName);
        assertNotNull(subscription);
        ((ManagedLedgerImpl)subscription.getCursor().getManagedLedger()).setEntriesAddedCounter(10L);
    }

    TopicStats topicStats = admin.topics().getPartitionedStats(topic, false);
    assertEquals(topicStats.subscriptions.get(subName).msgBacklog, 20);

    topicStats = admin.topics().getPartitionedStats(topic, false, true);
    assertEquals(topicStats.subscriptions.get(subName).msgBacklog, 1);
}
 

@Test
public void testConsumerBacklogEvictionWithAck() throws Exception {
    assertEquals(admin.namespaces().getBacklogQuotaMap("prop/ns-quota"),
            ConfigHelper.backlogQuotaMap(config));
    admin.namespaces().setBacklogQuota("prop/ns-quota",
            new BacklogQuota(10 * 1024, BacklogQuota.RetentionPolicy.consumer_backlog_eviction));
    PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()).build();

    final String topic1 = "persistent://prop/ns-quota/topic11";
    final String subName1 = "c11";
    final String subName2 = "c21";
    final int numMsgs = 20;

    Consumer<byte[]> consumer1 = client.newConsumer().topic(topic1).subscriptionName(subName1).subscribe();
    Consumer<byte[]> consumer2 = client.newConsumer().topic(topic1).subscriptionName(subName2).subscribe();
    org.apache.pulsar.client.api.Producer<byte[]> producer = client.newProducer().topic(topic1).create();
    byte[] content = new byte[1024];
    for (int i = 0; i < numMsgs; i++) {
        producer.send(content);
        // only one consumer acknowledges the message
        consumer1.acknowledge(consumer1.receive());
        consumer2.receive();
    }

    Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000);
    rolloverStats();

    TopicStats stats = admin.topics().getStats(topic1);
    assertTrue(stats.backlogSize <= 10 * 1024, "Storage size is [" + stats.storageSize + "]");
    client.close();
}
 

@Test
public void testConsumerBacklogEviction() throws Exception {
    assertEquals(admin.namespaces().getBacklogQuotaMap("prop/ns-quota"),
            ConfigHelper.backlogQuotaMap(config));
    admin.namespaces().setBacklogQuota("prop/ns-quota",
            new BacklogQuota(10 * 1024, BacklogQuota.RetentionPolicy.consumer_backlog_eviction));
    PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()).statsInterval(0, TimeUnit.SECONDS)
            .build();

    final String topic1 = "persistent://prop/ns-quota/topic1";
    final String subName1 = "c1";
    final String subName2 = "c2";
    final int numMsgs = 20;

    Consumer<byte[]> consumer1 = client.newConsumer().topic(topic1).subscriptionName(subName1).subscribe();
    Consumer<byte[]> consumer2 = client.newConsumer().topic(topic1).subscriptionName(subName2).subscribe();
    org.apache.pulsar.client.api.Producer<byte[]> producer = client.newProducer().topic(topic1).create();
    byte[] content = new byte[1024];
    for (int i = 0; i < numMsgs; i++) {
        producer.send(content);
        consumer1.receive();
        consumer2.receive();
    }

    Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000);
    rolloverStats();

    TopicStats stats = admin.topics().getStats(topic1);
    assertTrue(stats.backlogSize < 10 * 1024, "Storage size is [" + stats.storageSize + "]");
    client.close();
}
 

@Test
public void testAckStatsOnPartitionedTopicForExclusiveSubscription() throws PulsarAdminException, PulsarClientException, InterruptedException {
    final String topic = "persistent://my-property/my-ns/testAckStatsOnPartitionedTopicForExclusiveSubscription";
    admin.topics().createPartitionedTopic(topic, 3);
    Consumer<byte[]> consumer = pulsarClient.newConsumer()
            .topic(topic)
            .subscriptionType(SubscriptionType.Exclusive)
            .subscriptionName("sub")
            .subscribe();

    Producer<byte[]> producer = pulsarClient.newProducer()
            .topic(topic)
            .create();

    final int messages = 10;
    for (int i = 0; i < messages; i++) {
        producer.send(("message-" + i).getBytes());
    }

    int received = 0;
    for (int i = 0; i < messages; i++) {
        consumer.acknowledge(consumer.receive());
        received++;
    }
    Assert.assertEquals(messages, received);

    // wait acknowledge send
    Thread.sleep(2000);

    for (int i = 0; i < 3; i++) {
        TopicStats stats = admin.topics().getStats(topic + "-partition-" + i);
        Assert.assertEquals(stats.subscriptions.size(), 1);
        Assert.assertEquals(stats.subscriptions.entrySet().iterator().next().getValue().consumers.size(), 1);
        Assert.assertEquals(stats.subscriptions.entrySet().iterator().next().getValue().consumers.get(0).unackedMessages, 0);
    }
}
 

private void testProxy(String serviceUrl, String httpServiceUrl) throws Exception {
    final String tenant = "proxy-test-" + randomName(10);
    final String namespace = tenant + "/ns1";
    final String topic = "persistent://" + namespace + "/topic1";

    @Cleanup
    PulsarAdmin admin = PulsarAdmin.builder()
        .serviceHttpUrl(httpServiceUrl)
        .build();

    admin.tenants().createTenant(tenant,
            new TenantInfo(Collections.emptySet(), Collections.singleton(pulsarCluster.getClusterName())));

    admin.namespaces().createNamespace(namespace, Collections.singleton(pulsarCluster.getClusterName()));

    @Cleanup
    PulsarClient client = PulsarClient.builder()
        .serviceUrl(serviceUrl)
        .build();

    client.newConsumer()
            .topic(topic)
            .subscriptionName("sub1")
            .subscribe()
            .close();

    @Cleanup
    Producer<String> producer = client.newProducer(Schema.STRING)
            .topic(topic)
            .create();
    producer.send("content-0");
    producer.send("content-1");

    for (int i = 0; i < 10; i++) {
        // Ensure we can get the stats for the topic irrespective of which broker the proxy decides to connect to
        TopicStats stats = admin.topics().getStats(topic);
        assertEquals(stats.publishers.size(), 1);
    }
}
 

private static void checkSubscriptionsCleanup(String topic) throws Exception {
    try {
        ContainerExecResult result = pulsarCluster.getAnyBroker().execCmd(
                PulsarCluster.ADMIN_SCRIPT,
                "topics",
                "stats",
                topic);
        TopicStats topicStats = new Gson().fromJson(result.getStdout(), TopicStats.class);
        assertEquals(topicStats.subscriptions.size(), 0);

    } catch (ContainerExecException e) {
        fail("Command should have exited with non-zero");
    }
}
 

@Test
public void testPersistentTopicStats() {
    TopicStats topicStats = new TopicStats();
    topicStats.msgRateIn = 1;
    topicStats.msgThroughputIn = 1;
    topicStats.msgRateOut = 1;
    topicStats.msgThroughputOut = 1;
    topicStats.averageMsgSize = 1;
    topicStats.storageSize = 1;
    topicStats.publishers.add(new PublisherStats());
    topicStats.subscriptions.put("test_ns", new SubscriptionStats());
    topicStats.replication.put("test_ns", new ReplicatorStats());
    TopicStats target = new TopicStats();
    target.add(topicStats);
    assertEquals(topicStats.msgRateIn, 1.0);
    assertEquals(topicStats.msgThroughputIn, 1.0);
    assertEquals(topicStats.msgRateOut, 1.0);
    assertEquals(topicStats.msgThroughputOut, 1.0);
    assertEquals(topicStats.averageMsgSize, 1.0);
    assertEquals(topicStats.storageSize, 1);
    assertEquals(topicStats.publishers.size(), 1);
    assertEquals(topicStats.subscriptions.size(), 1);
    assertEquals(topicStats.replication.size(), 1);
    topicStats.reset();
    assertEquals(topicStats.msgRateIn, 0.0);
    assertEquals(topicStats.msgThroughputIn, 0.0);
    assertEquals(topicStats.msgRateOut, 0.0);
    assertEquals(topicStats.msgThroughputOut, 0.0);
    assertEquals(topicStats.averageMsgSize, 0.0);
    assertEquals(topicStats.storageSize, 0);
    assertEquals(topicStats.publishers.size(), 0);
    assertEquals(topicStats.subscriptions.size(), 0);
    assertEquals(topicStats.replication.size(), 0);
}
 

@Test
public void testPersistentTopicStatsAggregation() {
    TopicStats topicStats1 = new TopicStats();
    topicStats1.msgRateIn = 1;
    topicStats1.msgThroughputIn = 1;
    topicStats1.msgRateOut = 1;
    topicStats1.msgThroughputOut = 1;
    topicStats1.averageMsgSize = 1;
    topicStats1.storageSize = 1;
    topicStats1.publishers.add(new PublisherStats());
    topicStats1.subscriptions.put("test_ns", new SubscriptionStats());
    topicStats1.replication.put("test_ns", new ReplicatorStats());

    TopicStats topicStats2 = new TopicStats();
    topicStats2.msgRateIn = 1;
    topicStats2.msgThroughputIn = 2;
    topicStats2.msgRateOut = 3;
    topicStats2.msgThroughputOut = 4;
    topicStats2.averageMsgSize = 5;
    topicStats2.storageSize = 6;
    topicStats2.publishers.add(new PublisherStats());
    topicStats2.subscriptions.put("test_ns", new SubscriptionStats());
    topicStats2.replication.put("test_ns", new ReplicatorStats());

    TopicStats target = new TopicStats();
    target.add(topicStats1);
    target.add(topicStats2);

    assertEquals(target.msgRateIn, 2.0);
    assertEquals(target.msgThroughputIn, 3.0);
    assertEquals(target.msgRateOut, 4.0);
    assertEquals(target.msgThroughputOut, 5.0);
    assertEquals(target.averageMsgSize, 3.0);
    assertEquals(target.storageSize, 7);
    assertEquals(target.publishers.size(), 1);
    assertEquals(target.subscriptions.size(), 1);
    assertEquals(target.replication.size(), 1);
}
 

private void verifyBacklogInTopicStats(PersistentTopic persistentTopic, int expected) throws Exception {
    final AtomicLong backlog = new AtomicLong(0);
    retryStrategically(
        ((test) -> {
            backlog.set(persistentTopic.getStats(true).backlogSize);
            return backlog.get() == expected;
        }),
        5,
        200);

    if (log.isDebugEnabled()) {
        TopicStats topicStats = persistentTopic.getStats(true);
        log.info(" dump topicStats for topic : {}, storageSize: {}, backlogSize: {}, expected: {}",
            persistentTopic.getName(),
            topicStats.storageSize, topicStats.backlogSize, expected);

        topicStats.subscriptions.forEach((subname, substats) -> {
            log.debug(" dump sub: subname - {}, activeConsumerName {}, "
                    + "consumers {}, msgBacklog {}, unackedMessages {}.",
                subname,
                substats.activeConsumerName, substats.consumers,
                substats.msgBacklog, substats.unackedMessages);
        });

        persistentTopic.getManagedLedger().getCursors().forEach(cursor ->
            log.debug(" dump cursor: cursor - {}, durable: {}, numberEntryis: {},"
                    + " readPosition: {}, markdeletePosition: {}",
                cursor.getName(), cursor.isDurable(), cursor.getNumberOfEntries(),
                cursor.getReadPosition(), cursor.getMarkDeletedPosition()));
    }

    assertEquals(backlog.get(), expected);
}
 

public MessageId getPositionFromSubscription(String topic, MessageId defaultPosition) {
    try {
        TopicStats topicStats = admin.topics().getStats(topic);
        if (topicStats.subscriptions.containsKey(subscriptionName)) {
            SubscriptionStats subStats = topicStats.subscriptions.get(subscriptionName);
            if (subStats.consumers.size() != 0) {
                throw new RuntimeException("Subscription been actively used by other consumers, " +
                        "in this situation, the exactly-once semantics cannot be guaranteed.");
            } else {
                PersistentTopicInternalStats.CursorStats c =
                        admin.topics().getInternalStats(topic).cursors.get(subscriptionName);
                String[] ids = c.markDeletePosition.split(":", 2);
                long ledgerId = Long.parseLong(ids[0]);
                long entryIdInMarkDelete = Long.parseLong(ids[1]);
                // we are getting the next mid from sub position, if the entryId is -1,
                // it denotes we haven't read data from the ledger before,
                // therefore no need to skip the current entry for the next position
                long entryId = entryIdInMarkDelete == -1 ? -1 : entryIdInMarkDelete + 1;
                int partitionIdx = TopicName.getPartitionIndex(topic);
                return new MessageIdImpl(ledgerId, entryId, partitionIdx);
            }
        } else {
            // create sub on topic
            log.info("Setting up subscription {} on topic {} at position {}",
                subscriptionName, topic, defaultPosition);
            admin.topics().createSubscription(topic, subscriptionName, defaultPosition);
            log.info("Subscription {} on topic {} at position {} finished",
                subscriptionName, topic, defaultPosition);
            return defaultPosition;
        }
    } catch (PulsarAdminException e) {
        throw new RuntimeException("Failed to get stats for topic " + topic, e);
    }
}
 

@GET
@Path("{property}/{cluster}/{namespace}/{topic}/stats")
@ApiOperation(hidden = true, value = "Get the stats for the topic.")
@ApiResponses(value = {
        @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"),
        @ApiResponse(code = 403, message = "Don't have admin permission"),
        @ApiResponse(code = 404, message = "Topic does not exist") })
public TopicStats getStats(@PathParam("property") String property, @PathParam("cluster") String cluster,
        @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic,
        @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) {
    validateTopicName(property, cluster, namespace, encodedTopic);
    return internalGetStats(authoritative, false);
}
 

/**
 * verifies admin api command for non-persistent topic. It verifies: partitioned-topic, stats
 *
 * @throws Exception
 */
@Test
public void nonPersistentTopics() throws Exception {
    final String topicName = "nonPersistentTopic";

    final String persistentTopicName = "non-persistent://prop-xyz/ns1/" + topicName;
    // Force to create a topic
    publishMessagesOnTopic("non-persistent://prop-xyz/ns1/" + topicName, 0, 0);

    // create consumer and subscription
    @Cleanup
    PulsarClient client = PulsarClient.builder()
            .serviceUrl(pulsar.getWebServiceAddress())
            .statsInterval(0, TimeUnit.SECONDS)
            .build();
    Consumer<byte[]> consumer = client.newConsumer().topic(persistentTopicName).subscriptionName("my-sub")
            .subscribe();

    publishMessagesOnTopic("non-persistent://prop-xyz/ns1/" + topicName, 10, 0);

    TopicStats topicStats = admin.topics().getStats(persistentTopicName);
    assertEquals(topicStats.subscriptions.keySet(), Sets.newTreeSet(Lists.newArrayList("my-sub")));
    assertEquals(topicStats.subscriptions.get("my-sub").consumers.size(), 1);
    assertEquals(topicStats.publishers.size(), 0);

    PersistentTopicInternalStats internalStats = admin.topics().getInternalStats(persistentTopicName);
    assertEquals(internalStats.cursors.keySet(), Sets.newTreeSet(Lists.newArrayList("my-sub")));

    consumer.close();

    topicStats = admin.topics().getStats(persistentTopicName);
    assertTrue(topicStats.subscriptions.keySet().contains("my-sub"));
    assertEquals(topicStats.publishers.size(), 0);

    // test partitioned-topic
    final String partitionedTopicName = "non-persistent://prop-xyz/ns1/paritioned";
    assertEquals(admin.topics().getPartitionedTopicMetadata(partitionedTopicName).partitions, 0);
    admin.topics().createPartitionedTopic(partitionedTopicName, 5);
    assertEquals(admin.topics().getPartitionedTopicMetadata(partitionedTopicName).partitions, 5);
}
 

protected TopicStats internalGetStats(boolean authoritative, boolean getPreciseBacklog) {
    validateAdminAndClientPermission();
    if (topicName.isGlobal()) {
        validateGlobalNamespaceOwnership(namespaceName);
    }
    validateTopicOwnership(topicName, authoritative);
    Topic topic = getTopicReference(topicName);
    return topic.getStats(getPreciseBacklog);
}
 

protected void populateDimensionMap(Map<Metrics, List<TopicStats>> topicsStatsByDimensionMap,
        Metrics metrics, TopicStats destStats) {
    if (!topicsStatsByDimensionMap.containsKey(metrics)) {
        // create new list
        topicsStatsByDimensionMap.put(metrics, Lists.newArrayList(destStats));
    } else {
        // add to collection
        topicsStatsByDimensionMap.get(metrics).add(destStats);
    }

}
 

@Test(timeOut = 30000)
private void testPulsarSinkDLQ() throws Exception {
    final String namespacePortion = "io";
    final String replNamespace = tenant + "/" + namespacePortion;
    final String sourceTopic = "persistent://" + replNamespace + "/input";
    final String dlqTopic = sourceTopic+"-DLQ";
    final String sinkName = "PulsarSink-test";
    final String propertyKey = "key";
    final String propertyValue = "value";
    final String subscriptionName = "test-sub";
    admin.namespaces().createNamespace(replNamespace);
    Set<String> clusters = Sets.newHashSet(Lists.newArrayList("use"));
    admin.namespaces().setNamespaceReplicationClusters(replNamespace, clusters);
    // 1 create producer、DLQ consumer
    Producer<String> producer = pulsarClient.newProducer(Schema.STRING).topic(sourceTopic).create();
    Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING).topic(dlqTopic).subscriptionName(subscriptionName).subscribe();

    // 2 setup sink
    SinkConfig sinkConfig = createSinkConfig(tenant, namespacePortion, sinkName, sourceTopic, subscriptionName);
    sinkConfig.setNegativeAckRedeliveryDelayMs(1001L);
    sinkConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE);
    sinkConfig.setMaxMessageRetries(2);
    sinkConfig.setDeadLetterTopic(dlqTopic);
    sinkConfig.setInputSpecs(Collections.singletonMap(sourceTopic, ConsumerConfig.builder().receiverQueueSize(1000).build()));
    sinkConfig.setClassName(SinkForTest.class.getName());
    LocalRunner localRunner = LocalRunner.builder()
            .sinkConfig(sinkConfig)
            .clientAuthPlugin(AuthenticationTls.class.getName())
            .clientAuthParams(String.format("tlsCertFile:%s,tlsKeyFile:%s", TLS_CLIENT_CERT_FILE_PATH, TLS_CLIENT_KEY_FILE_PATH))
            .useTls(true)
            .tlsTrustCertFilePath(TLS_TRUST_CERT_FILE_PATH)
            .tlsAllowInsecureConnection(true)
            .tlsHostNameVerificationEnabled(false)
            .brokerServiceUrl(pulsar.getBrokerServiceUrlTls()).build();

    localRunner.start(false);

    retryStrategically((test) -> {
        try {
            TopicStats topicStats = admin.topics().getStats(sourceTopic);

            return topicStats.subscriptions.containsKey(subscriptionName)
                    && topicStats.subscriptions.get(subscriptionName).consumers.size() == 1
                    && topicStats.subscriptions.get(subscriptionName).consumers.get(0).availablePermits == 1000;

        } catch (PulsarAdminException e) {
            return false;
        }
    }, 50, 150);

    // 3 send message
    int totalMsgs = 10;
    for (int i = 0; i < totalMsgs; i++) {
        producer.newMessage().property(propertyKey, propertyValue).value("fail" + i).sendAsync();
    }

    //4 All messages should enter DLQ
    for (int i = 0; i < totalMsgs; i++) {
        Message<String> message = consumer.receive(10, TimeUnit.SECONDS);
        assertNotNull(message);
        assertEquals(message.getValue(), "fail" + i);
    }

    //clean up
    producer.close();
    consumer.close();
}