org.apache.kafka.connect.data.Values Java Examples

/**
 * Try to build a {@link SchemaBuilder} for a value of type {@link Object}
 * However, this will only build the schema only for known types, in case it can not return the precise SchemaBuilder type
 * it will return an optional {@link SchemaBuilder.Type}
 * @param value to return the SchemaBuilder for
 *
 * @return {@link SchemaBuilder} instance
 */
public static SchemaBuilder buildSchemaBuilderForType(final Object value) {
    Objects.requireNonNull(value);

    // gracefully try to infer the schema
    final Schema knownSchema = Values.inferSchema(value);

    if (knownSchema == null) {
        // let's now check for other types
        if (value instanceof Date) {
            return org.apache.kafka.connect.data.Date.builder();
        }
        if (value instanceof BigDecimal) {
            return Decimal.builder(((BigDecimal) value).scale());
        }
        // we re-check map and list since inferSchema function is not tolerant against map and list
        // for now we rely on inferSchema, however it makes some point to build a specific inferSchema method only for this connector
        if (value instanceof Map) {
            return new SchemaBuilder(Schema.Type.MAP);
        }
        if (value instanceof List) {
            return new SchemaBuilder(Schema.Type.ARRAY);
        }
        // if we do not fine any of schema out of the above, we just return an an optional byte schema
        return SchemaBuilder.bytes().optional();
    }
    return SchemaUtil.copySchemaBasics(knownSchema);
}
 

@Test
public void readFromRandomSource() throws Exception {
    System.setProperty("hazelcast.logging.type", "log4j");
    Properties randomProperties = new Properties();
    randomProperties.setProperty("name", "random-source-connector");
    randomProperties.setProperty("connector.class", "sasakitoa.kafka.connect.random.RandomSourceConnector");
    randomProperties.setProperty("generator.class", "sasakitoa.kafka.connect.random.generator.RandomInt");
    randomProperties.setProperty("tasks.max", "1");
    randomProperties.setProperty("messages.per.second", "1000");
    randomProperties.setProperty("topic", "test");
    randomProperties.setProperty("task.summary.enable", "true");

    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(KafkaConnectSources.connect(randomProperties))
            .withoutTimestamps()
            .map(record -> Values.convertToString(record.valueSchema(), record.value()))
            .writeTo(AssertionSinks.assertCollectedEventually(60,
                    list -> assertEquals(ITEM_COUNT, list.size())));

    JobConfig jobConfig = new JobConfig();
    jobConfig.addJar(Objects.requireNonNull(this.getClass()
                                                      .getClassLoader()
                                                      .getResource("random-connector-1.0-SNAPSHOT.jar"))
                                  .getPath()
    );

    Job job = createJetMember().newJob(pipeline, jobConfig);

    sleepAtLeastSeconds(5);

    try {
        job.join();
        fail("Job should have completed with an AssertionCompletedException, but completed normally");
    } catch (CompletionException e) {
        String errorMsg = e.getCause().getMessage();
        assertTrue("Job was expected to complete with AssertionCompletedException, but completed with: "
                + e.getCause(), errorMsg.contains(AssertionCompletedException.class.getName()));
    }
}
 

public static void main(String[] args) {
    JetInstance jet = JetBootstrap.getInstance();

    Configuration configuration = Configuration
            .create()
            .with("name", "mysql-demo-connector")
            .with("connector.class", "io.debezium.connector.mysql.MySqlConnector")
            /* begin connector properties */
            .with("database.hostname", "mysql")
            .with("database.port", "3306")
            .with("database.user", "debezium")
            .with("database.password", "dbz")
            .with("database.server.id", "184054")
            .with("database.server.name", "dbserver1")
            .with("database.whitelist", "inventory")
            .with("database.history.hazelcast.list.name", "test")
            .with("snapshot.mode", "schema_only")
            .build();

    Pipeline p = Pipeline.create();

    p.readFrom(DebeziumSources.cdc(configuration))
     .withoutTimestamps()
     .map(sourceRecord -> {
         String keyString = Values.convertToString(sourceRecord.keySchema(), sourceRecord.key());
         String valueString = Values.convertToString(sourceRecord.valueSchema(), sourceRecord.value());
         return Tuple2.tuple2(keyString, valueString);
     })
     .writeTo(Sinks.logger());

    jet.newJob(p).join();
}
 

@Override
@SuppressWarnings("unchecked")
public Object deserialize(AbstractType<?> abstractType, ByteBuffer bb) {
    Set<?> deserializedSet = (Set<?>) super.deserialize(abstractType, bb);
    List<?> deserializedList = (new ArrayList<>(deserializedSet));
    return Values.convertToList(getSchemaBuilder(abstractType).build(), deserializedList);
}
 

@Test
public void testTimeUUIDType() {
    UUID timeUUID = UUID.randomUUID();
    String expectedFixedUUID = Values.convertToString(CassandraTypeKafkaSchemaBuilders.UUID_TYPE, UuidUtil.asBytes(timeUUID));

    ByteBuffer serializedTimeUUID = TimeUUIDType.instance.decompose(timeUUID);

    Object deserializedTimeUUID = CassandraTypeDeserializer.deserialize(TimeUUIDType.instance, serializedTimeUUID);

    Assert.assertEquals(expectedFixedUUID, deserializedTimeUUID);
}
 

@Test
public void testUUIDType() {
    UUID uuid = UUID.randomUUID();
    String expectedFixedUUID = Values.convertToString(CassandraTypeKafkaSchemaBuilders.UUID_TYPE, UuidUtil.asBytes(uuid));

    ByteBuffer serializedUUID = UUIDType.instance.decompose(uuid);

    Object deserializedUUID = CassandraTypeDeserializer.deserialize(UUIDType.instance, serializedUUID);

    Assert.assertEquals(expectedFixedUUID, deserializedUUID);
}
 

@Override
Object convert(Header header) {
  return Values.convertToDate(header.schema(), header.value());
}
 

@Override
Object convert(Header header) {
  return Values.convertToTime(header.schema(), header.value());
}
 

@Override
Object convert(Header header) {
  return Values.convertToTimestamp(header.schema(), header.value());
}
 

@Override
Object convert(Header header) {
  return Values.convertToDecimal(header.schema(), header.value(), scale);
}
 

@Override
Object convert(Header header) {
  return Values.convertToLong(header.schema(), header.value());
}
 

@Override
Object convert(Header header) {
  return Values.convertToInteger(header.schema(), header.value());
}
 

@Override
Object convert(Header header) {
  return Values.convertToShort(header.schema(), header.value());
}
 

@Override
Object convert(Header header) {
  return Values.convertToByte(header.schema(), header.value());
}
 

@Override
Object convert(Header header) {
  return Values.convertToDouble(header.schema(), header.value());
}
 

@Override
Object convert(Header header) {
  return Values.convertToFloat(header.schema(), header.value());
}
 

@Override
Object convert(Header header) {
  return Values.convertToBoolean(header.schema(), header.value());
}
 

@Override
Object convert(Header header) {
  return Values.convertToString(header.schema(), header.value());
}
 

@Override
public Object deserialize(AbstractType<?> abstractType, ByteBuffer bb) {
    Object value = super.deserialize(abstractType, bb);
    byte[] bytes = UuidUtil.asBytes((java.util.UUID) value);
    return Values.convertToString(getSchemaBuilder(abstractType).build(), bytes);
}
 

@Override
@SuppressWarnings("unchecked")
public Object deserialize(AbstractType<?> abstractType, ByteBuffer bb) {
    List<?> deserializedList = (List<?>) super.deserialize(abstractType, bb);
    return Values.convertToList(getSchemaBuilder(abstractType).build(), deserializedList);
}
 

@Override
public Object deserialize(AbstractType<?> abstractType, ByteBuffer bb) {
    Map<?, ?> deserializedMap = (Map<?, ?>) super.deserialize(abstractType, bb);
    MapType<?, ?> mapType = (MapType<?, ?>) abstractType;
    return Values.convertToMap(getSchemaBuilder(mapType).build(), deserializedMap);
}
 

@Override
public Object deserialize(AbstractType<?> abstractType, ByteBuffer bb) {
    Object value = super.deserialize(abstractType, bb);
    byte[] bytes = UuidUtil.asBytes((java.util.UUID) value);
    return Values.convertToString(getSchemaBuilder(abstractType).build(), bytes);
}
 

@Test
public void readFromMongoDb() throws Exception {
    // populate initial data
    mongo.execInContainer("sh", "-c", "/usr/local/bin/init-inventory.sh");

    Configuration configuration = Configuration
            .create()
            .with("name", "mongodb-inventory-connector")
            .with("connector.class", "io.debezium.connector.mongodb.MongoDbConnector")
            /* begin connector properties */
            .with("mongodb.hosts", "rs0/" + mongo.getContainerIpAddress() + ":"
                    + mongo.getMappedPort(MongoDBContainer.MONGODB_PORT))
            .with("mongodb.name", "fullfillment")
            .with("mongodb.user", "debezium")
            .with("mongodb.password", "dbz")
            .with("mongodb.members.auto.discover", "false")
            .with("collection.whitelist", "inventory.*")
            .with("database.history.hazelcast.list.name", "test")
            .build();

    JetInstance jet = createJetMember();

    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(DebeziumSources.cdc(configuration))
            .withoutTimestamps()
            .map(record -> Values.convertToString(record.valueSchema(), record.value()))
            .writeTo(AssertionSinks.assertCollectedEventually(60,
                    list -> Assert.assertTrue(list.stream().anyMatch(s -> s.contains("Jason")))));

    JobConfig jobConfig = new JobConfig();
    jobConfig.addJarsInZip(Objects.requireNonNull(this.getClass()
                                                      .getClassLoader()
                                                      .getResource("debezium-connector-mongodb.zip")));

    Job job = jet.newJob(pipeline, jobConfig);
    assertJobStatusEventually(job, JobStatus.RUNNING);

    // update record
    mongo.execInContainer("sh", "-c", "/usr/local/bin/insertData.sh");

    try {
        job.join();
        Assert.fail("Job should have completed with an AssertionCompletedException, but completed normally");
    } catch (CompletionException e) {
        String errorMsg = e.getCause().getMessage();
        Assert.assertTrue("Job was expected to complete with AssertionCompletedException, " +
                "but completed with: " + e.getCause(), errorMsg.contains(AssertionCompletedException.class.getName()));
    }


}
 

@Test
public void readFromMsSql() throws Exception {
    execInContainer("setup.sql");
    assertTrueEventually(() -> {
        Container.ExecResult result = execInContainer("cdc.sql");
        assertContains(result.getStdout(), "already");
    });

    Configuration configuration = Configuration
            .create()
            .with("name", "mssql-inventory-connector")
            .with("connector.class", "io.debezium.connector.sqlserver.SqlServerConnector")
            /* begin connector properties */
            .with("tasks.max", "1")
            .with("database.hostname", mssql.getContainerIpAddress())
            .with("database.port", mssql.getMappedPort(MS_SQL_SERVER_PORT))
            .with("database.user", mssql.getUsername())
            .with("database.password", mssql.getPassword())
            .with("database.dbname", "MyDB")
            .with("database.server.name", "fulfillment")
            .with("table.whitelist", "inventory.customers")
            .with("database.history.hazelcast.list.name", "test")
            .build();

    JetInstance jet = createJetMember();

    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(DebeziumSources.cdc(configuration))
            .withoutTimestamps()
            .map(record -> Values.convertToString(record.valueSchema(), record.value()))
            .writeTo(AssertionSinks.assertCollectedEventually(60,
                    list -> assertTrue(list.stream().anyMatch(s -> s.contains("Anne Marie")))));

    JobConfig jobConfig = new JobConfig();
    jobConfig.addJarsInZip(Objects.requireNonNull(this.getClass()
                                                      .getClassLoader()
                                                      .getResource("debezium-connector-sqlserver.zip")));

    Job job = jet.newJob(pipeline, jobConfig);
    assertJobStatusEventually(job, JobStatus.RUNNING);

    sleepAtLeastSeconds(30);
    // update record
    try (Connection connection = DriverManager.getConnection(mssql.getJdbcUrl() + ";databaseName=MyDB",
            mssql.getUsername(), mssql.getPassword())) {
        connection.setSchema("inventory");
        PreparedStatement preparedStatement = connection.prepareStatement("update MyDB.inventory.customers set " +
                "first_name = 'Anne Marie' where id = 1001;");
        preparedStatement.executeUpdate();
    }

    try {
        job.join();
        fail("Job should have completed with an AssertionCompletedException, but completed normally");
    } catch (CompletionException e) {
        String errorMsg = e.getCause().getMessage();
        assertTrue("Job was expected to complete with AssertionCompletedException, but completed with: "
                + e.getCause(), errorMsg.contains(AssertionCompletedException.class.getName()));
    }


}
 

@Test
public void readFromMySql() throws Exception {
    // given
    Configuration configuration = Configuration
            .create()
            .with("name", "mysql-inventory-connector")
            .with("connector.class", "io.debezium.connector.mysql.MySqlConnector")
            /* begin connector properties */
            .with("database.hostname", mysql.getContainerIpAddress())
            .with("database.port", mysql.getMappedPort(MYSQL_PORT))
            .with("database.user", "debezium")
            .with("database.password", "dbz")
            .with("database.server.id", "184054")
            .with("database.server.name", "dbserver1")
            .with("database.whitelist", "inventory")
            .with("table.whitelist", "inventory.customers")
            .with("include.schema.changes", "false")
            .with("database.history.hazelcast.list.name", "test")
            .build();

    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(DebeziumSources.cdc(configuration))
            .withoutTimestamps()
            .map(record -> Values.convertToString(record.valueSchema(), record.value()))
            .filterUsingService(ServiceFactories.sharedService(context -> {
                Serde<EventRecord> serde = DebeziumSerdes.payloadJson(EventRecord.class);
                serde.configure(Collections.emptyMap(), false);
                return serde.deserializer();
            }, Deserializer::close), (deserializer, record) -> {
                EventRecord eventRecord = deserializer.deserialize("", record.getBytes());
                return eventRecord.isUpdate();
            })
            .writeTo(AssertionSinks.assertCollectedEventually(30,
                    list -> Assert.assertTrue(list.stream().anyMatch(s -> s.contains("Anne Marie")))));

    JobConfig jobConfig = new JobConfig();
    jobConfig.addJarsInZip(Objects.requireNonNull(this.getClass()
                                                      .getClassLoader()
                                                      .getResource("debezium-connector-mysql.zip")));

    // when
    JetInstance jet = createJetMember();
    Job job = jet.newJob(pipeline, jobConfig);
    assertJobStatusEventually(job, JobStatus.RUNNING);

    sleepAtLeastSeconds(10);
    // update a record
    try (Connection connection = DriverManager.getConnection(mysql.withDatabaseName("inventory").getJdbcUrl(),
            mysql.getUsername(), mysql.getPassword())) {
        PreparedStatement preparedStatement = connection
                .prepareStatement("UPDATE customers SET first_name='Anne Marie' WHERE id=1004;");
        preparedStatement.executeUpdate();
    }


    // then
    try {
        job.join();
        Assert.fail("Job should have completed with an AssertionCompletedException, but completed normally");
    } catch (CompletionException e) {
        String errorMsg = e.getCause().getMessage();
        Assert.assertTrue("Job was expected to complete with " +
                        "AssertionCompletedException, but completed with: " + e.getCause(),
                errorMsg.contains(AssertionCompletedException.class.getName()));
    }
}
 

@Test
public void readFromPostgres() throws Exception {
    Configuration configuration = Configuration
            .create()
            .with("name", "postgres-inventory-connector")
            .with("connector.class", "io.debezium.connector.postgresql.PostgresConnector")
            /* begin connector properties */
            .with("tasks.max", "1")
            .with("database.hostname", postgres.getContainerIpAddress())
            .with("database.port", postgres.getMappedPort(POSTGRESQL_PORT))
            .with("database.user", "postgres")
            .with("database.password", "postgres")
            .with("database.dbname", "postgres")
            .with("database.server.name", "dbserver1")
            .with("schema.whitelist", "inventory")
            .with("database.history.hazelcast.list.name", "test")
            .build();

    JetInstance jet = createJetMember();

    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(DebeziumSources.cdc(configuration))
            .withoutTimestamps()
            .map(record -> Values.convertToString(record.valueSchema(), record.value()))
            .writeTo(AssertionSinks.assertCollectedEventually(60,
                    list -> assertTrue(list.stream().anyMatch(s -> s.contains("Anne Marie")))));

    JobConfig jobConfig = new JobConfig();
    jobConfig.addJarsInZip(Objects.requireNonNull(this.getClass()
                                                      .getClassLoader()
                                                      .getResource("debezium-connector-postgres.zip")));

    Job job = jet.newJob(pipeline, jobConfig);
    assertJobStatusEventually(job, JobStatus.RUNNING);

    // update record
    try (Connection connection = DriverManager.getConnection(postgres.getJdbcUrl(), postgres.getUsername(),
            postgres.getPassword())) {
        connection.setSchema("inventory");
        PreparedStatement preparedStatement = connection.prepareStatement("update customers set " +
                "first_name = 'Anne Marie' where id = 1004;");
        preparedStatement.executeUpdate();
    }

    try {
        job.join();
        fail("Job should have completed with an AssertionCompletedException, but completed normally");
    } catch (CompletionException e) {
        String errorMsg = e.getCause().getMessage();
        assertTrue("Job was expected to complete with AssertionCompletedException, but completed with: "
                + e.getCause(), errorMsg.contains(AssertionCompletedException.class.getName()));
    }


}