Java Code Examples for org.apache.pulsar.client.api.Schema#encode()

The following examples show how to use org.apache.pulsar.client.api.Schema#encode() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: KeyValue.java    From pulsar with Apache License 2.0 6 votes vote down vote up 
/**
 * Encode a <tt>key</tt> and <tt>value</tt> pair into a bytes array.
 *
 * @param key key object to encode
 * @param keyWriter a writer to encode key object
 * @param value value object to encode
 * @param valueWriter a writer to encode value object
 * @return the encoded bytes array
 */
public static <K, V> byte[] encode(K key, Schema<K> keyWriter,
                                   V value, Schema<V> valueWriter) {
    byte [] keyBytes;
    if (key == null) {
        keyBytes = new byte[0];
    } else {
        keyBytes = keyWriter.encode(key);
    }

    byte [] valueBytes;
    if (value == null) {
        valueBytes = new byte[0];
    } else {
        valueBytes = valueWriter.encode(value);
    }
    ByteBuffer byteBuffer = ByteBuffer.allocate(4 + keyBytes.length + 4 + valueBytes.length);
    byteBuffer
        .putInt(key == null ? -1 : keyBytes.length)
        .put(keyBytes)
        .putInt(value == null ? -1 : valueBytes.length)
        .put(valueBytes);
    return byteBuffer.array();
}
Example 2
Source File: GenericSchemaImplTest.java    From pulsar with Apache License 2.0 6 votes vote down vote up 
private void testEncodeAndDecodeGenericRecord(Schema<Foo> encodeSchema,
                                              Schema<GenericRecord> decodeSchema) {
    int numRecords = 10;
    for (int i = 0; i < numRecords; i++) {
        Foo foo = newFoo(i);
        byte[] data = encodeSchema.encode(foo);

        log.info("Decoding : {}", new String(data, UTF_8));

        GenericRecord record;
        if (decodeSchema instanceof AutoConsumeSchema) {
            record = decodeSchema.decode(data, new byte[0]);
        } else {
            record = decodeSchema.decode(data);
        }
        verifyFooRecord(record, i);
    }
}
Example 3
Source File: KeyValueSchemaTest.java    From pulsar with Apache License 2.0 6 votes vote down vote up 
@Test
public void testAllowNullSchemaEncodeAndDecode() {
    Schema keyValueSchema = Schema.KeyValue(Foo.class, Bar.class);

    Bar bar = new Bar();
    bar.setField1(true);

    Foo foo = new Foo();
    foo.setField1("field1");
    foo.setField2("field2");
    foo.setField3(3);
    foo.setField4(bar);
    foo.setColor(Color.RED);

    byte[] encodeBytes = keyValueSchema.encode(new KeyValue(foo, bar));
    Assert.assertTrue(encodeBytes.length > 0);

    KeyValue<Foo, Bar> keyValue = (KeyValue<Foo, Bar>) keyValueSchema.decode(encodeBytes);
    Foo fooBack = keyValue.getKey();
    Bar barBack = keyValue.getValue();

    assertEquals(foo, fooBack);
    assertEquals(bar, barBack);
}
Example 4
Source File: KeyValueSchemaTest.java    From pulsar with Apache License 2.0 6 votes vote down vote up 
@Test
public void testNotAllowNullSchemaEncodeAndDecode() {
    Schema keyValueSchema = Schema.KeyValue(JSONSchema.of(SchemaDefinition.<Foo>builder().withPojo(Foo.class).withAlwaysAllowNull(false).build()),
            JSONSchema.of(SchemaDefinition.<Bar>builder().withPojo(Bar.class).withAlwaysAllowNull(false).build()));

    Bar bar = new Bar();
    bar.setField1(true);

    Foo foo = new Foo();
    foo.setField1("field1");
    foo.setField2("field2");
    foo.setField3(3);
    foo.setField4(bar);
    foo.setColor(Color.RED);

    byte[] encodeBytes = keyValueSchema.encode(new KeyValue(foo, bar));
    Assert.assertTrue(encodeBytes.length > 0);

    KeyValue<Foo, Bar> keyValue = (KeyValue<Foo, Bar>) keyValueSchema.decode(encodeBytes);
    Foo fooBack = keyValue.getKey();
    Bar barBack = keyValue.getValue();

    assertEquals(foo, fooBack);
    assertEquals(bar, barBack);
}
Example 5
Source File: PulsarFunctionsTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up 
protected Map<String, String> produceSchemaInsertMessagesToInputTopic(String inputTopicName,
                                                                      int numMessages,
                                                                      Schema<Foo> schema) throws Exception {
    @Cleanup
    PulsarClient client = PulsarClient.builder()
        .serviceUrl(pulsarCluster.getPlainTextServiceUrl())
        .build();

    @Cleanup
    Producer<Foo> producer = client.newProducer(schema)
        .topic(inputTopicName)
        .create();

    LinkedHashMap<String, String> kvs = new LinkedHashMap<>();
    for (int i = 0; i < numMessages; i++) {
        String key = "key-" + i;

        JdbcPostgresSinkTester.Foo obj = new JdbcPostgresSinkTester.Foo();
        obj.setField1("field1_insert_" + i);
        obj.setField2("field2_insert_" + i);
        obj.setField3(i);
        String value = new String(schema.encode(obj));
        Map<String, String> properties = Maps.newHashMap();
        properties.put("ACTION", "INSERT");

        kvs.put(key, value);
        kvs.put("ACTION", "INSERT");
        producer.newMessage()
                .properties(properties)
                .key(key)
                .value(obj)
                .send();
    }
    return kvs;
}
Example 6
Source File: PulsarFunctionsTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up 
protected Map<String, String> produceSchemaUpdateMessagesToInputTopic(String inputTopicName,
                                                                      int numMessages,
                                                                      Schema<Foo> schema) throws Exception {
    @Cleanup
    PulsarClient client = PulsarClient.builder()
            .serviceUrl(pulsarCluster.getPlainTextServiceUrl())
            .build();

    @Cleanup
    Producer<Foo> producer = client.newProducer(schema)
            .topic(inputTopicName)
            .create();

    LinkedHashMap<String, String> kvs = new LinkedHashMap<>();
    log.info("update start");
    for (int i = 0; i < numMessages; i++) {
        String key = "key-" + i;

        JdbcPostgresSinkTester.Foo obj = new JdbcPostgresSinkTester.Foo();
        obj.setField1("field1_insert_" + i);
        obj.setField2("field2_update_" + i);
        obj.setField3(i);
        String value = new String(schema.encode(obj));
        Map<String, String> properties = Maps.newHashMap();
        properties.put("ACTION", "UPDATE");

        kvs.put(key, value);
        kvs.put("ACTION", "UPDATE");
        producer.newMessage()
                .properties(properties)
                .key(key)
                .value(obj)
                .send();
    }
    log.info("update end");
    return kvs;
}
Example 7
Source File: PulsarFunctionsTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up 
protected Map<String, String> produceSchemaDeleteMessagesToInputTopic(String inputTopicName,
                                                                      int numMessages,
                                                                      Schema<Foo> schema) throws Exception {
    @Cleanup
    PulsarClient client = PulsarClient.builder()
            .serviceUrl(pulsarCluster.getPlainTextServiceUrl())
            .build();

    @Cleanup
    Producer<Foo> producer = client.newProducer(schema)
            .topic(inputTopicName)
            .create();

    LinkedHashMap<String, String> kvs = new LinkedHashMap<>();
    for (int i = 0; i < numMessages; i++) {
        String key = "key-" + i;

        JdbcPostgresSinkTester.Foo obj = new JdbcPostgresSinkTester.Foo();
        obj.setField1("field1_insert_" + i);
        obj.setField2("field2_update_" + i);
        obj.setField3(i);
        String value = new String(schema.encode(obj));
        Map<String, String> properties = Maps.newHashMap();
        properties.put("ACTION", "DELETE");

        kvs.put(key, value);
        kvs.put("ACTION", "DELETE");
        producer.newMessage()
                .properties(properties)
                .key(key)
                .value(obj)
                .send();
    }
    return kvs;
}
Example 8
Source File: GenericSchemaImplTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up 
private void testEncodeAndDecodeKeyValues(Schema<KeyValue<Foo, Foo>> encodeSchema,
                                          Schema<KeyValue<GenericRecord, GenericRecord>> decodeSchema) {
    int numRecords = 10;
    for (int i = 0; i < numRecords; i++) {
        Foo foo = newFoo(i);
        byte[] data = encodeSchema.encode(new KeyValue<>(foo, foo));

        KeyValue<GenericRecord, GenericRecord> kv = decodeSchema.decode(data, new byte[0]);
        verifyFooRecord(kv.getKey(), i);
        verifyFooRecord(kv.getValue(), i);
    }
}
Example 9
Source File: SupportVersioningKeyValueSchemaTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up 
@Test
public void testKeyValueVersioningEncodeDecode() {
    MultiVersionSchemaInfoProvider multiVersionSchemaInfoProvider = mock(MultiVersionSchemaInfoProvider.class);
    AvroSchema<SchemaTestUtils.Foo> fooSchema = AvroSchema.of(
            SchemaDefinition.<SchemaTestUtils.Foo>builder().withPojo(SchemaTestUtils.Foo.class).build());
    AvroSchema<SchemaTestUtils.Bar> barSchema = AvroSchema.of(
            SchemaDefinition.<SchemaTestUtils.Bar>builder().withPojo(SchemaTestUtils.Bar.class).build());
    Schema<KeyValue<SchemaTestUtils.Foo, SchemaTestUtils.Bar>> keyValueSchema = KeyValueSchema.of(
            fooSchema, barSchema);
    keyValueSchema.setSchemaInfoProvider(multiVersionSchemaInfoProvider);

    when(multiVersionSchemaInfoProvider.getSchemaByVersion(any(byte[].class)))
            .thenReturn(CompletableFuture.completedFuture(keyValueSchema.getSchemaInfo()));

    SchemaTestUtils.Bar bar = new SchemaTestUtils.Bar();
    bar.setField1(true);

    SchemaTestUtils.Foo foo = new SchemaTestUtils.Foo();
    foo.setField1("field1");
    foo.setField2("field2");
    foo.setField3(3);
    foo.setField4(bar);
    foo.setColor(SchemaTestUtils.Color.RED);

    byte[] encodeBytes = keyValueSchema.encode(new KeyValue(foo, bar));
    KeyValue<SchemaTestUtils.Foo, SchemaTestUtils.Bar> keyValue = keyValueSchema.decode(
            encodeBytes, new byte[10]);
    Assert.assertEquals(keyValue.getKey().getField1(), foo.getField1());
    Assert.assertEquals(keyValue.getKey().getField2(), foo.getField2());
    Assert.assertEquals(keyValue.getKey().getField3(), foo.getField3());
    Assert.assertEquals(keyValue.getKey().getField4(), foo.getField4());
    Assert.assertEquals(keyValue.getKey().getColor(), foo.getColor());
    Assert.assertTrue(keyValue.getValue().isField1());
    Assert.assertEquals(
            KeyValueEncodingType.valueOf(keyValueSchema.getSchemaInfo().getProperties().get("kv.encoding.type")),
            KeyValueEncodingType.INLINE);
}
Example 10
Source File: SupportVersioningKeyValueSchemaTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up 
@Test
public void testSeparateKeyValueVersioningEncodeDecode() {
    MultiVersionSchemaInfoProvider multiVersionSchemaInfoProvider = mock(MultiVersionSchemaInfoProvider.class);
    AvroSchema<SchemaTestUtils.Foo> fooSchema = AvroSchema.of(
            SchemaDefinition.<SchemaTestUtils.Foo>builder().withPojo(SchemaTestUtils.Foo.class).build());
    AvroSchema<SchemaTestUtils.Bar> barSchema = AvroSchema.of(
            SchemaDefinition.<SchemaTestUtils.Bar>builder().withPojo(SchemaTestUtils.Bar.class).build());
    Schema<KeyValue<SchemaTestUtils.Foo, SchemaTestUtils.Bar>> keyValueSchema = KeyValueSchema.of(
            fooSchema, barSchema, KeyValueEncodingType.SEPARATED);
    keyValueSchema.setSchemaInfoProvider(multiVersionSchemaInfoProvider);

    when(multiVersionSchemaInfoProvider.getSchemaByVersion(any(byte[].class)))
            .thenReturn(CompletableFuture.completedFuture(keyValueSchema.getSchemaInfo()));

    SchemaTestUtils.Bar bar = new SchemaTestUtils.Bar();
    bar.setField1(true);

    SchemaTestUtils.Foo foo = new SchemaTestUtils.Foo();
    foo.setField1("field1");
    foo.setField2("field2");
    foo.setField3(3);
    foo.setField4(bar);
    foo.setColor(SchemaTestUtils.Color.RED);

    byte[] encodeBytes = keyValueSchema.encode(new KeyValue(foo, bar));
    KeyValue<SchemaTestUtils.Foo, SchemaTestUtils.Bar> keyValue = ((KeyValueSchema)keyValueSchema).decode(
            fooSchema.encode(foo), encodeBytes, new byte[10]);
    Assert.assertTrue(keyValue.getValue().isField1());
    Assert.assertEquals(
            KeyValueEncodingType.valueOf(keyValueSchema.getSchemaInfo().getProperties().get("kv.encoding.type")),
            KeyValueEncodingType.SEPARATED);
}
Example 11
Source File: SupportVersioningKeyValueSchemaTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up 
@Test
public void testKeyValueDefaultVersioningEncodeDecode() {
    AvroSchema<SchemaTestUtils.Foo> fooSchema = AvroSchema.of(
            SchemaDefinition.<SchemaTestUtils.Foo>builder().withPojo(SchemaTestUtils.Foo.class).build());
    AvroSchema<SchemaTestUtils.Bar> barSchema = AvroSchema.of(
            SchemaDefinition.<SchemaTestUtils.Bar>builder().withPojo(SchemaTestUtils.Bar.class).build());
    Schema<KeyValue<SchemaTestUtils.Foo, SchemaTestUtils.Bar>> keyValueSchema = KeyValueSchema.of(
            fooSchema, barSchema);

    SchemaTestUtils.Bar bar = new SchemaTestUtils.Bar();
    bar.setField1(true);

    SchemaTestUtils.Foo foo = new SchemaTestUtils.Foo();
    foo.setField1("field1");
    foo.setField2("field2");
    foo.setField3(3);
    foo.setField4(bar);
    foo.setColor(SchemaTestUtils.Color.RED);

    byte[] encodeBytes = keyValueSchema.encode(new KeyValue(foo, bar));
    KeyValue<SchemaTestUtils.Foo, SchemaTestUtils.Bar> keyValue = keyValueSchema.decode(
            encodeBytes, new byte[10]);
    Assert.assertEquals(keyValue.getKey().getField1(), foo.getField1());
    Assert.assertEquals(keyValue.getKey().getField2(), foo.getField2());
    Assert.assertEquals(keyValue.getKey().getField3(), foo.getField3());
    Assert.assertEquals(keyValue.getKey().getField4(), foo.getField4());
    Assert.assertEquals(keyValue.getKey().getColor(), foo.getColor());
    Assert.assertTrue(keyValue.getValue().isField1());
    Assert.assertEquals(
            KeyValueEncodingType.valueOf(keyValueSchema.getSchemaInfo().getProperties().get("kv.encoding.type")),
            KeyValueEncodingType.INLINE);
}
Example 12
Source File: SupportVersioningKeyValueSchemaTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up 
@Test
public void testKeyValueLatestVersioningEncodeDecode() {
    AvroSchema<SchemaTestUtils.Foo> fooSchema = AvroSchema.of(
            SchemaDefinition.<SchemaTestUtils.Foo>builder().withPojo(SchemaTestUtils.Foo.class).build());
    AvroSchema<SchemaTestUtils.Bar> barSchema = AvroSchema.of(
            SchemaDefinition.<SchemaTestUtils.Bar>builder().withPojo(SchemaTestUtils.Bar.class).build());
    Schema<KeyValue<SchemaTestUtils.Foo, SchemaTestUtils.Bar>> keyValueSchema = KeyValueSchema.of(
            fooSchema, barSchema, KeyValueEncodingType.SEPARATED);

    SchemaTestUtils.Bar bar = new SchemaTestUtils.Bar();
    bar.setField1(true);

    SchemaTestUtils.Foo foo = new SchemaTestUtils.Foo();
    foo.setField1("field1");
    foo.setField2("field2");
    foo.setField3(3);
    foo.setField4(bar);
    foo.setColor(SchemaTestUtils.Color.RED);

    byte[] encodeBytes = keyValueSchema.encode(new KeyValue(foo, bar));
    KeyValue<SchemaTestUtils.Foo, SchemaTestUtils.Bar> keyValue = ((KeyValueSchema)keyValueSchema).decode(
            fooSchema.encode(foo), encodeBytes, new byte[10]);
    Assert.assertTrue(keyValue.getValue().isField1());
    Assert.assertEquals(
            KeyValueEncodingType.valueOf(keyValueSchema.getSchemaInfo().getProperties().get("kv.encoding.type")),
            KeyValueEncodingType.SEPARATED);
}
Example 13
Source File: BytesSchemaTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up 
private void testBytesSchema(Schema<byte[]> schema) {
    byte[] data = "hello world".getBytes(UTF_8);

    byte[] serializedData = schema.encode(data);
    assertSame(data, serializedData);

    byte[] deserializedData = schema.decode(serializedData);
    assertSame(data, deserializedData);
    ByteBuf byteBuf = ByteBufAllocator.DEFAULT.buffer(deserializedData.length);
    byteBuf.writeBytes(deserializedData);
    assertEquals(data, ((BytesSchema)schema).decode(byteBuf));

}
Example 14
Source File: KeyValueSchemaTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up 
@Test
public void testDefaultKeyValueEncodingTypeSchemaEncodeAndDecode() {
    AvroSchema<Foo> fooSchema = AvroSchema.of(SchemaDefinition.<Foo>builder().withPojo(Foo.class).build());
    AvroSchema<Bar> barSchema = AvroSchema.of(SchemaDefinition.<Bar>builder().withPojo(Bar.class).build());

    Schema<KeyValue<Foo, Bar>> keyValueSchema = Schema.KeyValue(fooSchema, barSchema);

    Bar bar = new Bar();
    bar.setField1(true);

    Foo foo = new Foo();
    foo.setField1("field1");
    foo.setField2("field2");
    foo.setField3(3);
    foo.setField4(bar);
    foo.setColor(Color.RED);

    // Check kv.encoding.type default not set value
    byte[] encodeBytes = keyValueSchema.encode(new KeyValue(foo, bar));
    Assert.assertTrue(encodeBytes.length > 0);

    KeyValue<Foo, Bar> keyValue = (KeyValue<Foo, Bar>) keyValueSchema.decode(encodeBytes);
    Foo fooBack = keyValue.getKey();
    Bar barBack = keyValue.getValue();

    assertEquals(foo, fooBack);
    assertEquals(bar, barBack);
}
Example 15
Source File: KeyValueSchemaTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up 
@Test
public void testInlineKeyValueEncodingTypeSchemaEncodeAndDecode() {

    AvroSchema<Foo> fooSchema = AvroSchema.of(SchemaDefinition.<Foo>builder().withPojo(Foo.class).build());
    AvroSchema<Bar> barSchema = AvroSchema.of(SchemaDefinition.<Bar>builder().withPojo(Bar.class).build());

    Schema<KeyValue<Foo, Bar>> keyValueSchema = Schema.KeyValue(fooSchema, barSchema, KeyValueEncodingType.INLINE);


    Bar bar = new Bar();
    bar.setField1(true);

    Foo foo = new Foo();
    foo.setField1("field1");
    foo.setField2("field2");
    foo.setField3(3);
    foo.setField4(bar);
    foo.setColor(Color.RED);

    // Check kv.encoding.type INLINE
    byte[] encodeBytes = keyValueSchema.encode(new KeyValue(foo, bar));
    Assert.assertTrue(encodeBytes.length > 0);
    KeyValue<Foo, Bar>  keyValue = (KeyValue<Foo, Bar>) keyValueSchema.decode(encodeBytes);
    Foo fooBack = keyValue.getKey();
    Bar barBack = keyValue.getValue();
    assertEquals(foo, fooBack);
    assertEquals(bar, barBack);

}
Example 16
Source File: KeyValueSchemaTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up 
@Test
public void testSeparatedKeyValueEncodingTypeSchemaEncodeAndDecode() {
    AvroSchema<Foo> fooSchema = AvroSchema.of(SchemaDefinition.<Foo>builder().withPojo(Foo.class).build());
    AvroSchema<Bar> barSchema = AvroSchema.of(SchemaDefinition.<Bar>builder().withPojo(Bar.class).build());

    Schema<KeyValue<Foo, Bar>> keyValueSchema = Schema.KeyValue(fooSchema, barSchema, KeyValueEncodingType.SEPARATED);

    Bar bar = new Bar();
    bar.setField1(true);

    Foo foo = new Foo();
    foo.setField1("field1");
    foo.setField2("field2");
    foo.setField3(3);
    foo.setField4(bar);
    foo.setColor(Color.RED);

    // Check kv.encoding.type SEPARATED
    byte[] encodeBytes = keyValueSchema.encode(new KeyValue(foo, bar));
    Assert.assertTrue(encodeBytes.length > 0);
    try {
        keyValueSchema.decode(encodeBytes);
        Assert.fail("This method cannot be used under this SEPARATED encoding type");
    } catch (SchemaSerializationException e) {
        Assert.assertTrue(e.getMessage().contains("This method cannot be used under this SEPARATED encoding type"));
    }
    KeyValue<Foo, Bar>  keyValue = ((KeyValueSchema)keyValueSchema).decode(fooSchema.encode(foo), encodeBytes, null);
    Foo fooBack = keyValue.getKey();
    Bar barBack = keyValue.getValue();
    assertEquals(foo, fooBack);
    assertEquals(bar, barBack);
}