org.apache.pulsar.client.api.Schema Java Examples

public PartitionedProducerImpl(PulsarClientImpl client, String topic, ProducerConfigurationData conf, int numPartitions,
        CompletableFuture<Producer<T>> producerCreatedFuture, Schema<T> schema, ProducerInterceptors interceptors) {
    super(client, topic, conf, producerCreatedFuture, schema, interceptors);
    this.producers = Lists.newArrayListWithCapacity(numPartitions);
    this.topicMetadata = new TopicMetadataImpl(numPartitions);
    this.routerPolicy = getMessageRouter();
    stats = client.getConfiguration().getStatsIntervalSeconds() > 0 ? new ProducerStatsRecorderImpl() : null;

    int maxPendingMessages = Math.min(conf.getMaxPendingMessages(),
            conf.getMaxPendingMessagesAcrossPartitions() / numPartitions);
    conf.setMaxPendingMessages(maxPendingMessages);
    start();

    // start track and auto subscribe partition increasement
    if (conf.isAutoUpdatePartitions()) {
        topicsPartitionChangedListener = new TopicsPartitionChangedListener();
        partitionsAutoUpdateTimeout = client.timer()
            .newTimeout(partitionsAutoUpdateTimerTask, 1, TimeUnit.MINUTES);
    }
}
 

@Test(dataProvider = "schemas")
public void allSchemasShouldSupportNull(Map<Schema, List<Object>> testData) {
    for (Schema<?> schema : testData.keySet()) {
        byte[] bytes = null;
        ByteBuf byteBuf =  null;
        try {
            assertNull(schema.encode(null),
                "Should support null in " + schema.getSchemaInfo().getName() + " serialization");
            assertNull(schema.decode(bytes),
                "Should support null in " + schema.getSchemaInfo().getName() + " deserialization");
            assertNull(((AbstractSchema) schema).decode(byteBuf),
                "Should support null in " + schema.getSchemaInfo().getName() + " deserialization");
        } catch (NullPointerException npe) {
            throw new NullPointerException("NPE when using schema " + schema + " : " + npe.getMessage());
        }
    }
}
 

/**
 * Required fields of Pulsar sink
 *
 * @param topic              Pulsar topic name to publish to
 * @param connectionSupplier Pulsar client supplier
 * @param extractValueFn     extracts the message value from the emitted items.
 * @param schemaSupplier     Pulsar messaging schema supplier.
 */
public PulsarSinkBuilder(
        @Nonnull String topic,
        @Nonnull SupplierEx<PulsarClient> connectionSupplier,
        @Nonnull SupplierEx<Schema<M>> schemaSupplier,
        @Nonnull FunctionEx<? super E, M> extractValueFn
) {
    checkSerializable(connectionSupplier, "connectionSupplier");
    checkSerializable(schemaSupplier, "schemaSupplier");
    checkSerializable(extractValueFn, "extractValueFn");

    this.topic = topic;
    this.producerConfig = getDefaultProducerConfig();
    this.connectionSupplier = connectionSupplier;
    this.schemaSupplier = schemaSupplier;
    this.extractValueFn = extractValueFn;
}
 

@Test
public void testAutoAvroSchema() {
    // configure encode schema
    Schema<Foo> encodeSchema = Schema.AVRO(Foo.class);

    // configure the schema info provider
    MultiVersionSchemaInfoProvider multiVersionGenericSchemaProvider = mock(MultiVersionSchemaInfoProvider.class);
    when(multiVersionGenericSchemaProvider.getSchemaByVersion(any(byte[].class)))
        .thenReturn(CompletableFuture.completedFuture(encodeSchema.getSchemaInfo()));

    // configure decode schema
    AutoConsumeSchema decodeSchema = new AutoConsumeSchema();
    decodeSchema.configureSchemaInfo(
        "test-topic", "topic", encodeSchema.getSchemaInfo()
    );
    decodeSchema.setSchemaInfoProvider(multiVersionGenericSchemaProvider);

    testEncodeAndDecodeGenericRecord(encodeSchema, decodeSchema);
}
 

@Test
public void testNotAllowNullAvroSchemaCreate() {
    AvroSchema<Foo> fooSchema = AvroSchema.of(SchemaDefinition.<Foo>builder().withPojo(Foo.class).withAlwaysAllowNull(false).build());
    AvroSchema<Bar> barSchema = AvroSchema.of(SchemaDefinition.<Bar>builder().withPojo(Bar.class).withAlwaysAllowNull(false).build());

    Schema<KeyValue<Foo, Bar>> keyValueSchema1 = Schema.KeyValue(fooSchema, barSchema);
    Schema<KeyValue<Foo, Bar>> keyValueSchema2 = Schema.KeyValue(AvroSchema.of(SchemaDefinition.<Foo>builder().withPojo(Foo.class).withAlwaysAllowNull(false).build()),
            AvroSchema.of(SchemaDefinition.<Bar>builder().withPojo(Bar.class).withAlwaysAllowNull(false).build()));

    assertEquals(keyValueSchema1.getSchemaInfo().getType(), SchemaType.KEY_VALUE);
    assertEquals(keyValueSchema2.getSchemaInfo().getType(), SchemaType.KEY_VALUE);

    assertEquals(((KeyValueSchema<Foo, Bar>) keyValueSchema1).getKeySchema().getSchemaInfo().getType(),
        SchemaType.AVRO);
    assertEquals(((KeyValueSchema<Foo, Bar>) keyValueSchema1).getValueSchema().getSchemaInfo().getType(),
        SchemaType.AVRO);
    assertEquals(((KeyValueSchema<Foo, Bar>) keyValueSchema2).getKeySchema().getSchemaInfo().getType(),
        SchemaType.AVRO);
    assertEquals(((KeyValueSchema<Foo, Bar>) keyValueSchema2).getValueSchema().getSchemaInfo().getType(),
        SchemaType.AVRO);

    String schemaInfo1 = new String(keyValueSchema1.getSchemaInfo().getSchema());
    String schemaInfo2 = new String(keyValueSchema2.getSchemaInfo().getSchema());
    assertEquals(schemaInfo1, schemaInfo2);
}
 

private ConsumerContext(
        @Nonnull ILogger logger,
        @Nonnull PulsarClient client,
        @Nonnull List<String> topics,
        @Nonnull Map<String, Object> consumerConfig,
        @Nonnull SupplierEx<Schema<M>> schemaSupplier,
        @Nonnull SupplierEx<BatchReceivePolicy> batchReceivePolicySupplier,
        @Nonnull FunctionEx<Message<M>, T> projectionFn
) throws PulsarClientException {

    this.logger = logger;
    this.projectionFn = projectionFn;
    this.client = client;
    this.consumer = client.newConsumer(schemaSupplier.get())
                          .topics(topics)
                          .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest)
                          .loadConf(consumerConfig)
                          .batchReceivePolicy(batchReceivePolicySupplier.get())
                          .subscriptionType(SubscriptionType.Shared)
                          .subscribe();
}
 

static <T> ConsumerImpl<T> newConsumerImpl(PulsarClientImpl client,
                                           String topic,
                                           ConsumerConfigurationData<T> conf,
                                           ExecutorService listenerExecutor,
                                           int partitionIndex,
                                           boolean hasParentConsumer,
                                           CompletableFuture<Consumer<T>> subscribeFuture,
                                           MessageId startMessageId,
                                           Schema<T> schema,
                                           ConsumerInterceptors<T> interceptors,
                                           boolean createTopicIfDoesNotExist) {
    if (conf.getReceiverQueueSize() == 0) {
        return new ZeroQueueConsumerImpl<>(client, topic, conf, listenerExecutor, partitionIndex, hasParentConsumer,
                subscribeFuture,
                startMessageId, schema, interceptors,
                createTopicIfDoesNotExist);
    } else {
        return new ConsumerImpl<>(client, topic, conf, listenerExecutor, partitionIndex, hasParentConsumer,
                subscribeFuture, startMessageId, 0 /* rollback time in sec to start msgId */,
                schema, interceptors, createTopicIfDoesNotExist);
    }
}
 

/**
 * Verify that JavaInstance does support functions that output Void type.
 */
@Test
public void testVoidOutputClasses() throws Exception {
    PulsarSinkConfig pulsarConfig = getPulsarConfigs();
    // set type to void
    pulsarConfig.setTypeClassName(Void.class.getName());
    PulsarSink pulsarSink = new PulsarSink(getPulsarClient(), pulsarConfig, new HashMap<>(), mock(ComponentStatsManager.class), Thread.currentThread().getContextClassLoader());

    try {
        Schema schema = pulsarSink.initializeSchema();
        assertNull(schema);
    } catch (Exception ex) {
        ex.printStackTrace();
        assertNull(ex);
        fail();
    }
}
 

@DataProvider(name = "schemas")
public static Object[][] schemas() {
    return new Object[][] {
        {
            Schema.STRING.getSchemaInfo(), UTF8_SCHEMA_INFO
        },
        {
            Schema.INT32.getSchemaInfo(), INT32_SCHEMA_INFO
        },
        {
            KeyValueSchemaInfoTest.FOO_SCHEMA.getSchemaInfo(), FOO_SCHEMA_INFO
        },
        {
            KeyValueSchemaInfoTest.BAR_SCHEMA.getSchemaInfo(), BAR_SCHEMA_INFO
        },
        {
            Schema.KeyValue(
                KeyValueSchemaInfoTest.FOO_SCHEMA,
                KeyValueSchemaInfoTest.BAR_SCHEMA,
                KeyValueEncodingType.SEPARATED
            ).getSchemaInfo(),
            KV_SCHEMA_INFO
        }
    };
}
 

@Test(dataProvider = "schemas")
public void allSchemasShouldRoundtripInput(Map<Schema, List<Object>> testData) {
    for (Map.Entry<Schema, List<Object>> test : testData.entrySet()) {
        log.info("Test schema {}", test.getKey());
        for (Object value : test.getValue()) {
            log.info("Encode : {}", value);
            try {
                assertEquals(value,
                    test.getKey().decode(test.getKey().encode(value)),
                    "Should get the original " + test.getKey().getSchemaInfo().getName() +
                        " after serialization and deserialization");
            } catch (NullPointerException npe) {
                throw new NullPointerException("NPE when using schema " + test.getKey()
                    + " : " + npe.getMessage());
            }
        }
    }
}
 

private void publishMessagesOnPersistentTopic(String topicName, int messages, int startIdx,
                                              boolean nullValue) throws Exception {
    Producer<byte[]> producer = pulsarClient.newProducer(Schema.BYTES)
        .topic(topicName)
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.SinglePartition)
        .create();

    for (int i = startIdx; i < (messages + startIdx); i++) {
        if (nullValue) {
            producer.send(null);
        } else {
            String message = "message-" + i;
            producer.send(message.getBytes());
        }
    }

    producer.close();
}
 

@Test
public void createReplicatedSubscription() throws Exception {
    String topic = "createReplicatedSubscription-" + System.nanoTime();

    @Cleanup
    Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING)
            .topic(topic)
            .subscriptionName("sub1")
            .replicateSubscriptionState(true)
            .subscribe();

    TopicStats stats = admin.topics().getStats(topic);
    assertTrue(stats.subscriptions.get("sub1").isReplicated);

    admin.topics().unload(topic);

    // Check that subscription is still marked replicated after reloading
    stats = admin.topics().getStats(topic);
    assertTrue(stats.subscriptions.get("sub1").isReplicated);
}
 

public static Schema<?> initializeCustomSchema(String schemaClassName, ClassLoader clsLoader, Class<?> typeArg,
                                               boolean input) {
    Schema<?> schema = createInstance(schemaClassName, clsLoader, Schema.class);

    Class<?>[] inputSerdeTypeArgs = TypeResolver.resolveRawArguments(Schema.class, schema.getClass());
    if (input) {
        checkArgument(typeArg.isAssignableFrom(inputSerdeTypeArgs[0]),
                "Inconsistent types found between function type and schema type: "
                        + " function type = " + typeArg + " should be assignable from "
                        + inputSerdeTypeArgs[0]);
    } else {
        checkArgument(inputSerdeTypeArgs[0].isAssignableFrom(typeArg),
                "Inconsistent types found between function type and schema type: "
                        + " schema type = " + inputSerdeTypeArgs[0] + " should be assignable from "
                        + typeArg);
    }
    return schema;
}
 

@Test
public void testDeleteNamespace() throws Exception {
    try (PulsarAdmin admin = buildAdminClient("admin")) {
        log.info("Creating tenant");
        admin.tenants().createTenant("tenant1",
                                     new TenantInfo(ImmutableSet.of("admin"), ImmutableSet.of("test")));
        log.info("Creating namespace, and granting perms to user1");
        admin.namespaces().createNamespace("tenant1/ns1", ImmutableSet.of("test"));
        admin.namespaces().grantPermissionOnNamespace("tenant1/ns1", "user1", ImmutableSet.of(AuthAction.produce));

        log.info("user1 produces some messages");
        try (PulsarClient client = buildClient("user1");
             Producer<String> producer = client.newProducer(Schema.STRING).topic("tenant1/ns1/foobar").create()) {
            producer.send("foobar");
        }

        log.info("Deleting the topic");
        admin.topics().delete("tenant1/ns1/foobar", true);

        log.info("Deleting namespace");
        admin.namespaces().deleteNamespace("tenant1/ns1");
    }
}
 

public void testPublishAndConsume(String serviceUrl, boolean isPersistent) throws Exception {
    String topicName = generateTopicName("testpubconsume", isPersistent);

    int numMessages = 10;

    try (PulsarClient client = PulsarClient.builder()
        .serviceUrl(serviceUrl)
        .build()) {

        try (Consumer<String> consumer = client.newConsumer(Schema.STRING)
            .topic(topicName)
            .subscriptionName("my-sub")
            .subscribe()) {

            try (Producer<String> producer = client.newProducer(Schema.STRING)
                .topic(topicName)
                .create()) {

                for (int i = 0; i < numMessages; i++) {
                    producer.send("smoke-message-" + i);
                }
            }

            for (int i = 0; i < numMessages; i++) {
                Message<String> m = consumer.receive();
                assertEquals("smoke-message-" + i, m.getValue());
            }
        }
    }
}
 

/**
 * Create a wrapper of type {@link ProducerInterceptor} that will delegate all work to underlying Kafka's interceptor.
 *
 * @param kafkaProducerInterceptor Underlying instance of {@link org.apache.kafka.clients.producer.ProducerInterceptor<K, V>}
 *                                 that this wrapper will delegate work to.
 * @param keySerializer            {@link Serializer} used to serialize Kafka {@link ProducerRecord#key}.
 * @param valueSerializer          {@link Serializer} used to serialize Kafka {@link ProducerRecord#value}.
 * @param topic                    Topic this {@link ProducerInterceptor} will be associated to.
 */
public KafkaProducerInterceptorWrapper(org.apache.kafka.clients.producer.ProducerInterceptor<K, V> kafkaProducerInterceptor,
                                       Schema<K> keySchema,
                                       Schema<V> valueSchema,
                                       String topic) {
    this.kafkaProducerInterceptor = kafkaProducerInterceptor;
    this.keySchema = keySchema;
    this.valueSchema = valueSchema;
    this.topic = topic;
}
 

@Test(dataProvider = "schemas")
public void testAllSchemas(Map<Schema, List<Object>> schemas) {
    for (Map.Entry<Schema, List<Object>> keyEntry : schemas.entrySet()) {
        for (Map.Entry<Schema, List<Object>> valueEntry : schemas.entrySet()) {
            testEncodeDecodeKeyValue(
                keyEntry.getKey(),
                valueEntry.getKey(),
                keyEntry.getValue(),
                valueEntry.getValue()
            );
        }
    }
}
 

@Test
public void testProducer() throws Exception {
    @Cleanup
    PulsarClient client = internalSetUpForClient(true, proxyService.getServiceUrlTls());
    @Cleanup
    Producer<byte[]> producer = client.newProducer(Schema.BYTES)
            .topic("persistent://sample/test/local/topic" + System.currentTimeMillis())
            .create();

    for (int i = 0; i < 10; i++) {
        producer.send("test".getBytes());
    }
}
 

@Test
public void testInboundConnection() throws Exception {
    LOG.info("Creating producer 1");
    @Cleanup
    PulsarClient client1 = PulsarClient.builder()
            .serviceUrl(proxyService.getServiceUrl())
            .operationTimeout(1000, TimeUnit.MILLISECONDS)
            .build();

    @Cleanup
    Producer<byte[]> producer1 = client1.newProducer(Schema.BYTES).topic("persistent://sample/test/local/producer-topic-1").create();

    LOG.info("Creating producer 2");
    @Cleanup
    PulsarClient client2 = PulsarClient.builder()
            .serviceUrl(proxyService.getServiceUrl())
            .operationTimeout(1000, TimeUnit.MILLISECONDS)
            .build();

    Assert.assertEquals(ProxyService.rejectedConnections.get(), 0.0d);
    try {
        @Cleanup
        Producer<byte[]> producer2 = client2.newProducer(Schema.BYTES).topic("persistent://sample/test/local/producer-topic-1").create();
        producer2.send("Message 1".getBytes());
        Assert.fail("Should have failed since max num of connections is 2 and the first producer used them all up - one for discovery and other for producing.");
    } catch (Exception ex) {
        // OK
    }
    // should add retry count since retry every 100ms and operation timeout is set to 1000ms
    Assert.assertEquals(ProxyService.rejectedConnections.get(), 5.0d);
}
 

public static <T extends com.google.protobuf.GeneratedMessageV3> Schema<T> newProtobufSchema(
    SchemaDefinition schemaDefinition) {
    return catchExceptions(
        () -> (Schema<T>) getStaticMethod(
            "org.apache.pulsar.client.impl.schema.ProtobufSchema", "of", SchemaDefinition.class)
            .invoke(null, schemaDefinition));
}
 

private void testProxy(String serviceUrl, String httpServiceUrl) throws Exception {
    final String tenant = "proxy-test-" + randomName(10);
    final String namespace = tenant + "/ns1";
    final String topic = "persistent://" + namespace + "/topic1";

    @Cleanup
    PulsarAdmin admin = PulsarAdmin.builder()
        .serviceHttpUrl(httpServiceUrl)
        .build();

    admin.tenants().createTenant(tenant,
            new TenantInfo(Collections.emptySet(), Collections.singleton(pulsarCluster.getClusterName())));

    admin.namespaces().createNamespace(namespace, Collections.singleton(pulsarCluster.getClusterName()));

    @Cleanup
    PulsarClient client = PulsarClient.builder()
        .serviceUrl(serviceUrl)
        .build();

    client.newConsumer()
            .topic(topic)
            .subscriptionName("sub1")
            .subscribe()
            .close();

    @Cleanup
    Producer<String> producer = client.newProducer(Schema.STRING)
            .topic(topic)
            .create();
    producer.send("content-0");
    producer.send("content-1");

    for (int i = 0; i < 10; i++) {
        // Ensure we can get the stats for the topic irrespective of which broker the proxy decides to connect to
        TopicStats stats = admin.topics().getStats(topic);
        assertEquals(stats.publishers.size(), 1);
    }
}
 

@Test
public void testDefaultKeyValueEncodingTypeSchemaEncodeAndDecode() {
    AvroSchema<Foo> fooSchema = AvroSchema.of(SchemaDefinition.<Foo>builder().withPojo(Foo.class).build());
    AvroSchema<Bar> barSchema = AvroSchema.of(SchemaDefinition.<Bar>builder().withPojo(Bar.class).build());

    Schema<KeyValue<Foo, Bar>> keyValueSchema = Schema.KeyValue(fooSchema, barSchema);

    Bar bar = new Bar();
    bar.setField1(true);

    Foo foo = new Foo();
    foo.setField1("field1");
    foo.setField2("field2");
    foo.setField3(3);
    foo.setField4(bar);
    foo.setColor(Color.RED);

    // Check kv.encoding.type default not set value
    byte[] encodeBytes = keyValueSchema.encode(new KeyValue(foo, bar));
    Assert.assertTrue(encodeBytes.length > 0);

    KeyValue<Foo, Bar> keyValue = (KeyValue<Foo, Bar>) keyValueSchema.decode(encodeBytes);
    Foo fooBack = keyValue.getKey();
    Bar barBack = keyValue.getValue();

    assertEquals(foo, fooBack);
    assertEquals(bar, barBack);
}
 

@Test(dataProvider = "encodingTypes")
public void encodeDecodeNestedKeyValueSchemaInfo(KeyValueEncodingType encodingType) {
    Schema<KeyValue<String, Bar>> nestedSchema =
        Schema.KeyValue(Schema.STRING, BAR_SCHEMA, KeyValueEncodingType.INLINE);
    Schema<KeyValue<Foo, KeyValue<String, Bar>>> kvSchema = Schema.KeyValue(
        FOO_SCHEMA,
        nestedSchema,
        encodingType
    );
    SchemaInfo kvSchemaInfo = kvSchema.getSchemaInfo();
    assertEquals(
        DefaultImplementation.decodeKeyValueEncodingType(kvSchemaInfo),
        encodingType);

    SchemaInfo encodedSchemaInfo =
        DefaultImplementation.encodeKeyValueSchemaInfo(
            FOO_SCHEMA,
            nestedSchema,
            encodingType);
    assertEquals(encodedSchemaInfo, kvSchemaInfo);
    assertEquals(
        DefaultImplementation.decodeKeyValueEncodingType(encodedSchemaInfo),
        encodingType);

    KeyValue<SchemaInfo, SchemaInfo> schemaInfoKeyValue =
        DefaultImplementation.decodeKeyValueSchemaInfo(kvSchemaInfo);

    assertEquals(schemaInfoKeyValue.getKey(), FOO_SCHEMA.getSchemaInfo());
    assertEquals(schemaInfoKeyValue.getValue().getType(), SchemaType.KEY_VALUE);
    KeyValue<SchemaInfo, SchemaInfo> nestedSchemaInfoKeyValue =
        DefaultImplementation.decodeKeyValueSchemaInfo(schemaInfoKeyValue.getValue());

    assertEquals(nestedSchemaInfoKeyValue.getKey(), Schema.STRING.getSchemaInfo());
    assertEquals(nestedSchemaInfoKeyValue.getValue(), BAR_SCHEMA.getSchemaInfo());
}
 

/**
 * Validates proxy connection stats api.
 * 
 * @throws Exception
 */
@Test
public void testConnectionsStats() throws Exception {
    final String topicName1 = "persistent://sample/test/local/connections-stats";
    PulsarClient client = PulsarClient.builder().serviceUrl(proxyService.getServiceUrl()).build();
    Producer<byte[]> producer = client.newProducer(Schema.BYTES).topic(topicName1).enableBatching(false)
            .messageRoutingMode(MessageRoutingMode.SinglePartition).create();

    // Create a consumer directly attached to broker
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName1).subscriptionName("my-sub").subscribe();

    int totalMessages = 10;
    for (int i = 0; i < totalMessages; i++) {
        producer.send("test".getBytes());
    }

    for (int i = 0; i < totalMessages; i++) {
        Message<byte[]> msg = consumer.receive(1, TimeUnit.SECONDS);
        checkNotNull(msg);
        consumer.acknowledge(msg);
    }

    Client httpClient = ClientBuilder.newClient(new ClientConfig().register(LoggingFeature.class));
    Response r = httpClient.target(proxyWebServer.getServiceUri()).path("/proxy-stats/connections").request()
            .get();
    Assert.assertEquals(r.getStatus(), Response.Status.OK.getStatusCode());
    String response = r.readEntity(String.class).trim();
    List<ConnectionStats> connectionStats = new Gson().fromJson(response, new TypeToken<List<ConnectionStats>>() {
    }.getType());

    assertNotNull(connectionStats);

    consumer.close();
    client.close();
}
 

@SuppressWarnings("unchecked")
private <T> Schema<T> newSchemaInstance(String topic, Class<T> clazz, ConsumerConfig conf, boolean input, ClassLoader classLoader) {
    // The schemaTypeOrClassName can represent multiple thing, either a schema type, a schema class name or a ser-de
    // class name.
    String schemaTypeOrClassName = conf.getSchemaType();
    if (StringUtils.isEmpty(schemaTypeOrClassName) || DEFAULT_SERDE.equals(schemaTypeOrClassName)) {
        // No preferred schema was provided, auto-discover schema or fallback to defaults
        return newSchemaInstance(clazz, getSchemaTypeOrDefault(topic, clazz));
    }

    SchemaType schemaType = null;
    try {
        schemaType = SchemaType.valueOf(schemaTypeOrClassName.toUpperCase());
    } catch (IllegalArgumentException e) {
        // schemaType is not referring to builtin type
    }

    if (schemaType != null) {
        // The parameter passed was indeed a valid builtin schema type
        return newSchemaInstance(clazz, schemaType, conf);
    }

    // At this point, the string can represent either a schema or serde class name. Create an instance and
    // check if it complies with either interface

    // First try with Schema
    try {
        return (Schema<T>) InstanceUtils.initializeCustomSchema(schemaTypeOrClassName,
                classLoader, clazz, input);
    } catch (Throwable t) {
        // Now try with Serde or just fail
        SerDe<T> serDe = (SerDe<T>) InstanceUtils.initializeSerDe(schemaTypeOrClassName,
                classLoader, clazz, input);
        return new SerDeSchema<>(serDe);
    }
}
 

/**
 * Publish from Java and consume from Python
 */
@Test(dataProvider = "ServiceUrls")
public void testJavaPublishPythonConsume(String serviceUrl) throws Exception {
    String nsName = generateNamespaceName();
    pulsarCluster.createNamespace(nsName);

    String topicName = generateTopicName(nsName, "testJavaPublishPythonConsume", true);

    @Cleanup
    PulsarClient client = PulsarClient.builder()
            .serviceUrl(serviceUrl)
            .build();

    // Create subscription to retain data
    client.newConsumer(Schema.JSON(Example1.class))
            .topic(topicName)
            .subscriptionName("my-subscription")
            .subscribe()
            .close();

    @Cleanup
    Producer<Example1> producer = client.newProducer(Schema.JSON(Example1.class))
            .topic(topicName)
            .create();

    Example1 e1 = new Example1();
    e1.setX(1);
    e1.setY(2L);
    producer.send(e1);

    // Verify Python can receive the typed message

    ContainerExecResult res = pulsarCluster.getAnyBroker()
            .execCmd("/pulsar/examples/python-examples/consumer_schema.py", "pulsar://localhost:6650", topicName);
    assertEquals(res.getExitCode(), 0);
}
 

@Test
public void testPartitions() throws Exception {
    @Cleanup
    PulsarClient client = internalSetUpForClient(true, proxyService.getServiceUrlTls());
    String topicName = "persistent://sample/test/local/partitioned-topic" + System.currentTimeMillis();
    TenantInfo tenantInfo = createDefaultTenantInfo();
    admin.tenants().createTenant("sample", tenantInfo);
    admin.topics().createPartitionedTopic(topicName, 2);

    @Cleanup
    Producer<byte[]> producer = client.newProducer(Schema.BYTES).topic(topicName)
            .messageRoutingMode(MessageRoutingMode.RoundRobinPartition).create();

    // Create a consumer directly attached to broker
    @Cleanup
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName)
            .subscriptionName("my-sub").subscribe();

    for (int i = 0; i < 10; i++) {
        producer.send("test".getBytes());
    }

    for (int i = 0; i < 10; i++) {
        Message<byte[]> msg = consumer.receive(1, TimeUnit.SECONDS);
        checkNotNull(msg);
    }
}
 

public static <T> MultiTopicsConsumerImpl<T> createPartitionedConsumer(PulsarClientImpl client,
                                                                       ConsumerConfigurationData<T> conf,
                                                                       ExecutorService listenerExecutor,
                                                                       CompletableFuture<Consumer<T>> subscribeFuture,
                                                                       int numPartitions,
                                                                       Schema<T> schema, ConsumerInterceptors<T> interceptors) {
    checkArgument(conf.getTopicNames().size() == 1, "Should have only 1 topic for partitioned consumer");

    // get topic name, then remove it from conf, so constructor will create a consumer with no topic.
    ConsumerConfigurationData cloneConf = conf.clone();
    String topicName = cloneConf.getSingleTopic();
    cloneConf.getTopicNames().remove(topicName);

    CompletableFuture<Consumer> future = new CompletableFuture<>();
    MultiTopicsConsumerImpl consumer = new MultiTopicsConsumerImpl(client, topicName, cloneConf, listenerExecutor,
            future, schema, interceptors, true /* createTopicIfDoesNotExist */);

    future.thenCompose(c -> ((MultiTopicsConsumerImpl)c).subscribeAsync(topicName, numPartitions))
        .thenRun(()-> subscribeFuture.complete(consumer))
        .exceptionally(e -> {
            log.warn("Failed subscription for createPartitionedConsumer: {} {}, e:{}",
                topicName, numPartitions,  e);
            subscribeFuture.completeExceptionally(
                PulsarClientException.wrap(((Throwable) e).getCause(), String.format("Failed to subscribe %s with %d partitions", topicName, numPartitions)));
            return null;
        });
    return consumer;
}
 

private <T> CompletableFuture<Consumer<T>> patternTopicSubscribeAsync(ConsumerConfigurationData<T> conf,
        Schema<T> schema, ConsumerInterceptors<T> interceptors) {
    String regex = conf.getTopicsPattern().pattern();
    Mode subscriptionMode = convertRegexSubscriptionMode(conf.getRegexSubscriptionMode());
    TopicName destination = TopicName.get(regex);
    NamespaceName namespaceName = destination.getNamespaceObject();

    CompletableFuture<Consumer<T>> consumerSubscribedFuture = new CompletableFuture<>();
    lookup.getTopicsUnderNamespace(namespaceName, subscriptionMode)
        .thenAccept(topics -> {
            if (log.isDebugEnabled()) {
                log.debug("Get topics under namespace {}, topics.size: {}", namespaceName.toString(), topics.size());
                topics.forEach(topicName ->
                    log.debug("Get topics under namespace {}, topic: {}", namespaceName.toString(), topicName));
            }

            List<String> topicsList = topicsPatternFilter(topics, conf.getTopicsPattern());
            conf.getTopicNames().addAll(topicsList);
            ConsumerBase<T> consumer = new PatternMultiTopicsConsumerImpl<T>(conf.getTopicsPattern(),
                PulsarClientImpl.this,
                conf,
                externalExecutorProvider.getExecutor(),
                consumerSubscribedFuture,
                schema, subscriptionMode, interceptors);

            synchronized (consumers) {
                consumers.put(consumer, Boolean.TRUE);
            }
        })
        .exceptionally(ex -> {
            log.warn("[{}] Failed to get topics under namespace", namespaceName);
            consumerSubscribedFuture.completeExceptionally(ex);
            return null;
        });

    return consumerSubscribedFuture;
}
 

@Test
public void testBatchMessageIndexAckForSharedSubscription() throws PulsarClientException, ExecutionException, InterruptedException {
    final String topic = "testBatchMessageIndexAckForSharedSubscription";

    @Cleanup
    Consumer<Integer> consumer = pulsarClient.newConsumer(Schema.INT32)
        .topic(topic)
        .subscriptionName("sub")
        .receiverQueueSize(100)
        .subscriptionType(SubscriptionType.Shared)
        .ackTimeout(1, TimeUnit.SECONDS)
        .subscribe();

    @Cleanup
    Producer<Integer> producer = pulsarClient.newProducer(Schema.INT32)
        .topic(topic)
        .batchingMaxPublishDelay(50, TimeUnit.MILLISECONDS)
        .create();

    final int messages = 100;
    List<CompletableFuture<MessageId>> futures = new ArrayList<>(messages);
    for (int i = 0; i < messages; i++) {
        futures.add(producer.sendAsync(i));
    }
    FutureUtil.waitForAll(futures).get();

    for (int i = 0; i < messages; i++) {
        if (i % 2 == 0) {
            consumer.acknowledge(consumer.receive());
        }
    }

    List<Message<Integer>> received = new ArrayList<>(messages);
    for (int i = 0; i < messages; i++) {
        received.add(consumer.receive());
    }

    Assert.assertEquals(received.size(), 100);
}