io.debezium.config.Configuration Java Examples

The following examples show how to use io.debezium.config.Configuration. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: PulsarDatabaseHistoryTest.java    From pulsar with Apache License 2.0 6 votes vote down vote up
@Test
public void testExists() {
    // happy path
    testHistoryTopicContent(true);
    assertTrue(history.exists());

    // Set history to use dummy topic
    Configuration config = Configuration.create()
        .with(PulsarDatabaseHistory.SERVICE_URL, brokerUrl.toString())
        .with(PulsarDatabaseHistory.TOPIC, "persistent://my-property/my-ns/dummytopic")
        .with(DatabaseHistory.NAME, "my-db-history")
        .with(DatabaseHistory.SKIP_UNPARSEABLE_DDL_STATEMENTS, true)
        .build();

    history.configure(config, null, DatabaseHistoryListener.NOOP, true);
    history.start();

    // dummytopic should not exist yet
    assertFalse(history.exists());
}
 
Example #2
Source File: Db2ConnectorIT.java    From debezium-incubator with Apache License 2.0 6 votes vote down vote up
@Test
@FixFor("DBZ-1242")
public void testEmptySchemaWarningAfterApplyingFilters() throws Exception {
    // This captures all logged messages, allowing us to verify log message was written.
    final LogInterceptor logInterceptor = new LogInterceptor();

    Configuration config = TestHelper.defaultConfig()
            .with(Db2ConnectorConfig.SNAPSHOT_MODE, SnapshotMode.INITIAL)
            .with(Db2ConnectorConfig.TABLE_WHITELIST, "my_products")
            .build();

    start(Db2Connector.class, config);
    assertConnectorIsRunning();
    waitForAvailableRecords(100, TimeUnit.MILLISECONDS);

    stopConnector(value -> assertThat(logInterceptor.containsWarnMessage(NO_MONITORED_TABLES_WARNING)).isTrue());
}
 
Example #3
Source File: StreamingDatatypesIT.java    From debezium-incubator with Apache License 2.0 6 votes vote down vote up
@Before
public void before() throws Exception {
    setConsumeTimeout(TestHelper.defaultMessageConsumerPollTimeout(), TimeUnit.SECONDS);
    dropTables();
    initializeConnectorTestFramework();
    Testing.Files.delete(TestHelper.DB_HISTORY_PATH);

    Configuration config = connectorConfig()
            .build();

    start(OracleConnector.class, config);
    assertConnectorIsRunning();

    waitForSnapshotToBeCompleted(TestHelper.CONNECTOR_NAME, TestHelper.SERVER_NAME);
    createTables();
}
 
Example #4
Source File: CassandraConnectorConfigTest.java    From debezium-incubator with Apache License 2.0 6 votes vote down vote up
@Test
public void testDefaultConfigs() {
    Properties props = new Properties();
    CassandraConnectorConfig config = new CassandraConnectorConfig(Configuration.from(props));
    assertEquals(CassandraConnectorConfig.DEFAULT_SNAPSHOT_CONSISTENCY, config.snapshotConsistencyLevel().name().toUpperCase());
    assertEquals(CassandraConnectorConfig.DEFAULT_HTTP_PORT, config.httpPort());
    assertArrayEquals(CassandraConnectorConfig.DEFAULT_CASSANDRA_HOST.split(","), config.cassandraHosts());
    assertEquals(CassandraConnectorConfig.DEFAULT_CASSANDRA_PORT, config.cassandraPort());
    assertEquals(CassandraConnectorConfig.DEFAULT_MAX_QUEUE_SIZE, config.maxQueueSize());
    assertEquals(CassandraConnectorConfig.DEFAULT_MAX_BATCH_SIZE, config.maxBatchSize());
    assertEquals(CassandraConnectorConfig.DEFAULT_POLL_INTERVAL_MS, config.pollIntervalMs().toMillis());
    assertEquals(CassandraConnectorConfig.DEFAULT_MAX_OFFSET_FLUSH_SIZE, config.maxOffsetFlushSize());
    assertEquals(CassandraConnectorConfig.DEFAULT_OFFSET_FLUSH_INTERVAL_MS, config.offsetFlushIntervalMs().toMillis());
    assertEquals(CassandraConnectorConfig.DEFAULT_SCHEMA_POLL_INTERVAL_MS, config.schemaPollIntervalMs().toMillis());
    assertEquals(CassandraConnectorConfig.DEFAULT_CDC_DIR_POLL_INTERVAL_MS, config.cdcDirPollIntervalMs().toMillis());
    assertEquals(CassandraConnectorConfig.DEFAULT_SNAPSHOT_POLL_INTERVAL_MS, config.snapshotPollIntervalMs().toMillis());
    assertEquals(CassandraConnectorConfig.DEFAULT_COMMIT_LOG_POST_PROCESSING_ENABLED, config.postProcessEnabled());
    assertEquals(CassandraConnectorConfig.DEFAULT_COMMIT_LOG_TRANSFER_CLASS, config.getCommitLogTransfer().getClass().getName());
    assertFalse(config.cassandraSslEnabled());
    assertFalse(config.tombstonesOnDelete());
    assertEquals(CassandraConnectorConfig.SnapshotMode.INITIAL, config.snapshotMode());
    assertEquals(CassandraConnectorConfig.DEFAULT_LATEST_COMMIT_LOG_ONLY, config.latestCommitLogOnly());
}
 
Example #5
Source File: QueueProcessorTest.java    From debezium-incubator with Apache License 2.0 6 votes vote down vote up
@Test
public void testProcessTombstoneRecords() throws Exception {
    doNothing().when(emitter).emit(any());

    int recordSize = 5;
    ChangeEventQueue<Event> queue = context.getQueue();
    for (int i = 0; i < recordSize; i++) {
        CassandraConnectorConfig config = new CassandraConnectorConfig(Configuration.from(new Properties()));
        SourceInfo sourceInfo = new SourceInfo(config, DatabaseDescriptor.getClusterName(),
                new OffsetPosition("CommitLog-6-123.log", i),
                new KeyspaceTable(TEST_KEYSPACE, "cdc_table"), false,
                Conversions.toInstantFromMicros(System.currentTimeMillis() * 1000));
        Record record = new TombstoneRecord(sourceInfo, new RowData(), Schema.INT32_SCHEMA);
        queue.enqueue(record);
    }

    assertEquals(recordSize, queue.totalCapacity() - queue.remainingCapacity());
    queueProcessor.process();
    verify(emitter, times(recordSize)).emit(any());
    assertEquals(queue.totalCapacity(), queue.remainingCapacity());
}
 
Example #6
Source File: QueueProcessorTest.java    From debezium-incubator with Apache License 2.0 6 votes vote down vote up
@Test
public void testProcessChangeRecords() throws Exception {
    doNothing().when(emitter).emit(any());

    int recordSize = 5;
    ChangeEventQueue<Event> queue = context.getQueue();
    for (int i = 0; i < recordSize; i++) {
        CassandraConnectorConfig config = new CassandraConnectorConfig(Configuration.from(new Properties()));
        SourceInfo sourceInfo = new SourceInfo(config, DatabaseDescriptor.getClusterName(),
                new OffsetPosition("CommitLog-6-123.log", i),
                new KeyspaceTable(TEST_KEYSPACE, "cdc_table"), false,
                Conversions.toInstantFromMicros(System.currentTimeMillis() * 1000));
        Record record = new ChangeRecord(sourceInfo, new RowData(), Schema.INT32_SCHEMA, Schema.INT32_SCHEMA, Record.Operation.INSERT, false);
        queue.enqueue(record);
    }

    assertEquals(recordSize, queue.totalCapacity() - queue.remainingCapacity());
    queueProcessor.process();
    verify(emitter, times(recordSize)).emit(any());
    assertEquals(queue.totalCapacity(), queue.remainingCapacity());
}
 
Example #7
Source File: TestHelper.java    From debezium-incubator with Apache License 2.0 6 votes vote down vote up
public static OracleConnection testConnection() {
    Configuration config = testConfig().build();
    Configuration jdbcConfig = config.subset("database.", true);

    OracleConnection jdbcConnection = new OracleConnection(jdbcConfig, new OracleConnectionFactory());
    try {
        jdbcConnection.setAutoCommit(false);
    }
    catch (SQLException e) {
        throw new RuntimeException(e);
    }

    String pdbName = new OracleConnectorConfig(config).getPdbName();

    if (pdbName != null) {
        jdbcConnection.setSessionToPdb(pdbName);
    }

    return jdbcConnection;
}
 
Example #8
Source File: TestHelper.java    From debezium-incubator with Apache License 2.0 6 votes vote down vote up
public static OracleConnection adminConnection() {
    Configuration config = adminConfig().build();
    Configuration jdbcConfig = config.subset("database.", true);

    OracleConnection jdbcConnection = new OracleConnection(jdbcConfig, new OracleConnectionFactory());
    try {
        jdbcConnection.setAutoCommit(false);
    }
    catch (SQLException e) {
        throw new RuntimeException(e);
    }

    String pdbName = new OracleConnectorConfig(config).getPdbName();

    if (pdbName != null) {
        jdbcConnection.setSessionToPdb(pdbName);
    }

    return jdbcConnection;
}
 
Example #9
Source File: PulsarDatabaseHistory.java    From pulsar with Apache License 2.0 6 votes vote down vote up
@Override
public void configure(
        Configuration config,
        HistoryRecordComparator comparator,
        DatabaseHistoryListener listener,
        boolean useCatalogBeforeSchema) {
    super.configure(config, comparator, listener, useCatalogBeforeSchema);
    if (!config.validateAndRecord(ALL_FIELDS, logger::error)) {
        throw new IllegalArgumentException("Error configuring an instance of "
            + getClass().getSimpleName() + "; check the logs for details");
    }
    this.topicName = config.getString(TOPIC);
    this.serviceUrl = config.getString(SERVICE_URL);
    // Copy the relevant portions of the configuration and add useful defaults ...
    this.dbHistoryName = config.getString(DatabaseHistory.NAME, UUID.randomUUID().toString());

    log.info("Configure to store the debezium database history {} to pulsar topic {} at {}",
        dbHistoryName, topicName, serviceUrl);
}
 
Example #10
Source File: OracleConnectorIT.java    From debezium-incubator with Apache License 2.0 5 votes vote down vote up
@Test
public void shouldReadChangeStreamForTableCreatedWhileStreaming() throws Exception {
    TestHelper.dropTable(connection, "debezium.customer2");

    Configuration config = TestHelper.defaultConfig()
            .with(RelationalDatabaseConnectorConfig.TABLE_WHITELIST, "DEBEZIUM\\.CUSTOMER2")
            .build();

    start(OracleConnector.class, config);
    assertConnectorIsRunning();

    waitForSnapshotToBeCompleted(TestHelper.CONNECTOR_NAME, TestHelper.SERVER_NAME);

    String ddl = "create table debezium.customer2 (" +
            "  id numeric(9,0) not null, " +
            "  name varchar2(1000), " +
            "  score decimal(6, 2), " +
            "  registered timestamp, " +
            "  primary key (id)" +
            ")";

    connection.execute(ddl);
    connection.execute("GRANT SELECT ON debezium.customer2 to " + TestHelper.CONNECTOR_USER);

    connection.execute("INSERT INTO debezium.customer2 VALUES (2, 'Billie-Bob', 1234.56, TO_DATE('2018/02/22', 'yyyy-mm-dd'))");
    connection.execute("COMMIT");

    SourceRecords records = consumeRecordsByTopic(1);

    List<SourceRecord> testTableRecords = records.recordsForTopic("server1.DEBEZIUM.CUSTOMER2");
    assertThat(testTableRecords).hasSize(1);

    VerifyRecord.isValidInsert(testTableRecords.get(0), "ID", 2);
    Struct after = (Struct) ((Struct) testTableRecords.get(0).value()).get("after");
    assertThat(after.get("ID")).isEqualTo(2);
    assertThat(after.get("NAME")).isEqualTo("Billie-Bob");
    assertThat(after.get("SCORE")).isEqualTo(BigDecimal.valueOf(1234.56));
    assertThat(after.get("REGISTERED")).isEqualTo(toMicroSecondsSinceEpoch(LocalDateTime.of(2018, 2, 22, 0, 0, 0)));
}
 
Example #11
Source File: Db2ConnectorIT.java    From debezium-incubator with Apache License 2.0 5 votes vote down vote up
@Test
public void blacklistTable() throws Exception {
    final int RECORDS_PER_TABLE = 5;
    final int TABLES = 1;
    final int ID_START = 10;
    final Configuration config = TestHelper.defaultConfig()
            .with(Db2ConnectorConfig.SNAPSHOT_MODE, SnapshotMode.INITIAL)
            .with(Db2ConnectorConfig.TABLE_BLACKLIST, "db2inst1.tablea")
            .build();
    connection.execute(
            "INSERT INTO tableb VALUES(1, 'b')");

    start(Db2Connector.class, config);
    assertConnectorIsRunning();

    // Wait for snapshot completion
    consumeRecordsByTopic(1);

    TestHelper.enableDbCdc(connection);
    connection.execute("UPDATE ASNCDC.IBMSNAP_REGISTER SET STATE = 'A' WHERE SOURCE_OWNER = 'DB2INST1'");
    TestHelper.refreshAndWait(connection);

    for (int i = 0; i < RECORDS_PER_TABLE; i++) {
        final int id = ID_START + i;
        connection.execute(
                "INSERT INTO tablea VALUES(" + id + ", 'a')");
        connection.execute(
                "INSERT INTO tableb VALUES(" + id + ", 'b')");
    }

    TestHelper.refreshAndWait(connection);

    final SourceRecords records = consumeRecordsByTopic(RECORDS_PER_TABLE * TABLES);
    final List<SourceRecord> tableA = records.recordsForTopic("testdb.DB2INST1.TABLEA");
    final List<SourceRecord> tableB = records.recordsForTopic("testdb.DB2INST1.TABLEB");
    assertThat(tableA == null || tableA.isEmpty()).isTrue();
    assertThat(tableB).hasSize(RECORDS_PER_TABLE);

    stopConnector();
}
 
Example #12
Source File: DebeziumCDCWithJet.java    From hazelcast-jet-demos with Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) {
    JetInstance jet = JetBootstrap.getInstance();

    Configuration configuration = Configuration
            .create()
            .with("name", "mysql-demo-connector")
            .with("connector.class", "io.debezium.connector.mysql.MySqlConnector")
            /* begin connector properties */
            .with("database.hostname", "mysql")
            .with("database.port", "3306")
            .with("database.user", "debezium")
            .with("database.password", "dbz")
            .with("database.server.id", "184054")
            .with("database.server.name", "dbserver1")
            .with("database.whitelist", "inventory")
            .with("database.history.hazelcast.list.name", "test")
            .with("snapshot.mode", "schema_only")
            .build();

    Pipeline p = Pipeline.create();

    p.readFrom(DebeziumSources.cdc(configuration))
     .withoutTimestamps()
     .map(sourceRecord -> {
         String keyString = Values.convertToString(sourceRecord.keySchema(), sourceRecord.key());
         String valueString = Values.convertToString(sourceRecord.valueSchema(), sourceRecord.value());
         return Tuple2.tuple2(keyString, valueString);
     })
     .writeTo(Sinks.logger());

    jet.newJob(p).join();
}
 
Example #13
Source File: TestHelper.java    From debezium-incubator with Apache License 2.0 5 votes vote down vote up
public static JdbcConfiguration adminJdbcConfig() {
    return JdbcConfiguration.copy(Configuration.fromSystemProperties("database."))
            .withDefault(JdbcConfiguration.DATABASE, "testdb")
            .withDefault(JdbcConfiguration.HOSTNAME, "localhost")
            .withDefault(JdbcConfiguration.PORT, 50000)
            .withDefault(JdbcConfiguration.USER, "db2inst1")
            .withDefault(JdbcConfiguration.PASSWORD, "admin")
            .build();
}
 
Example #14
Source File: TestHelper.java    From debezium-incubator with Apache License 2.0 5 votes vote down vote up
public static JdbcConfiguration defaultJdbcConfig() {
    return JdbcConfiguration.copy(Configuration.fromSystemProperties("database."))
            .withDefault(JdbcConfiguration.DATABASE, TEST_DATABASE)
            .withDefault(JdbcConfiguration.HOSTNAME, "localhost")
            .withDefault(JdbcConfiguration.PORT, 50000)
            .withDefault(JdbcConfiguration.USER, "db2inst1")
            .withDefault(JdbcConfiguration.PASSWORD, "admin")
            .build();
}
 
Example #15
Source File: TestHelper.java    From debezium-incubator with Apache License 2.0 5 votes vote down vote up
/**
 * Returns a default configuration suitable for most test cases. Can be amended/overridden in individual tests as
 * needed.
 */
public static Configuration.Builder defaultConfig() {
    JdbcConfiguration jdbcConfiguration = defaultJdbcConfig();
    Configuration.Builder builder = Configuration.create();

    jdbcConfiguration.forEach(
            (field, value) -> builder.with(Db2ConnectorConfig.DATABASE_CONFIG_PREFIX + field, value));

    return builder.with(RelationalDatabaseConnectorConfig.SERVER_NAME, "testdb")
            .with(Db2ConnectorConfig.DATABASE_HISTORY, FileDatabaseHistory.class)
            .with(FileDatabaseHistory.FILE_PATH, DB_HISTORY_PATH)
            .with(RelationalDatabaseConnectorConfig.INCLUDE_SCHEMA_CHANGES, false);
}
 
Example #16
Source File: OracleConnectorConfig.java    From debezium-incubator with Apache License 2.0 5 votes vote down vote up
public OracleConnectorConfig(Configuration config) {
    super(OracleConnector.class, config, config.getString(SERVER_NAME), new SystemTablesPredicate(), x -> x.schema() + "." + x.table(), true);

    this.databaseName = config.getString(DATABASE_NAME);
    this.pdbName = config.getString(PDB_NAME);
    this.xoutServerName = config.getString(XSTREAM_SERVER_NAME);
    this.snapshotMode = SnapshotMode.parse(config.getString(SNAPSHOT_MODE));
    this.tablenameCaseInsensitive = config.getBoolean(TABLENAME_CASE_INSENSITIVE);
    this.oracleVersion = OracleVersion.parse(config.getString(ORACLE_VERSION));
}
 
Example #17
Source File: OracleConnectorConfig.java    From debezium-incubator with Apache License 2.0 5 votes vote down vote up
/**
 * Validate the time.precision.mode configuration.
 *
 * If {@code adaptive} is specified, this option has the potential to cause overflow which is why the
 * option was deprecated and no longer supported for this connector.
 */
private static int validateSnapshotMode(Configuration config, Field field, ValidationOutput problems) {
    if (config.hasKey(SNAPSHOT_MODE.name())) {
        final String snapshotMode = config.getString(SNAPSHOT_MODE.name());
        if (SnapshotMode.INITIAL_SCHEMA_ONLY.value.equals(snapshotMode)) {
            // this will be logged as ERROR, but returning 0 doesn't prevent start-up
            problems.accept(SNAPSHOT_MODE, snapshotMode,
                    "The 'initial_schema_only' snapshot.mode is no longer supported and will be removed in a future revision. Use 'schema_only' instead.");
            return 0;
        }
    }

    // Everything checks out ok.
    return 0;
}
 
Example #18
Source File: SourceInfoTest.java    From debezium-incubator with Apache License 2.0 5 votes vote down vote up
@Before
public void beforeEach() {
    final OracleConnectorConfig connectorConfig = new OracleConnectorConfig(
            Configuration.create()
                    .with(OracleConnectorConfig.SERVER_NAME, "serverX")
                    .with(OracleConnectorConfig.DATABASE_NAME, "mydb")
                    .build());
    source = new SourceInfo(connectorConfig);
    source.setSourceTime(Instant.now());
    source.setTableId(new TableId("c", "s", "t"));
}
 
Example #19
Source File: OracleConnectorIT.java    From debezium-incubator with Apache License 2.0 5 votes vote down vote up
@Test
public void shouldStreamAfterRestart() throws Exception {
    Configuration config = TestHelper.defaultConfig()
            .with(RelationalDatabaseConnectorConfig.TABLE_WHITELIST, "DEBEZIUM\\.CUSTOMER")
            .build();

    // Testing.Print.enable();
    int expectedRecordCount = 0;
    connection.execute("INSERT INTO debezium.customer VALUES (1, 'Billie-Bob', 1234.56, TO_DATE('2018/02/22', 'yyyy-mm-dd'))");
    connection.execute("INSERT INTO debezium.customer VALUES (2, 'Bruce', 2345.67, null)");
    connection.execute("COMMIT");
    expectedRecordCount += 2;

    start(OracleConnector.class, config);
    assertConnectorIsRunning();

    SourceRecords records = consumeRecordsByTopic(expectedRecordCount);
    List<SourceRecord> testTableRecords = records.recordsForTopic("server1.DEBEZIUM.CUSTOMER");
    assertThat(testTableRecords).hasSize(expectedRecordCount);

    expectedRecordCount = 30;
    connection.setAutoCommit(false);
    sendTxBatch(expectedRecordCount, 100);
    sendTxBatch(expectedRecordCount, 200);

    stopConnector();
    final int OFFSET = 300;
    for (int i = OFFSET; i < expectedRecordCount + OFFSET; i++) {
        connection.executeWithoutCommitting(String.format("INSERT INTO debezium.customer VALUES (%s, 'Brian%s', 2345.67, null)", i, i));
    }
    connection.connection().commit();

    start(OracleConnector.class, config);
    assertConnectorIsRunning();

    assertTxBatch(expectedRecordCount, 300);
    sendTxBatch(expectedRecordCount, 400);
    sendTxBatch(expectedRecordCount, 500);
}
 
Example #20
Source File: OracleConnectorIT.java    From debezium-incubator with Apache License 2.0 5 votes vote down vote up
@Test
public void shouldStreamAfterRestartAfterSnapshot() throws Exception {
    Configuration config = TestHelper.defaultConfig()
            .with(RelationalDatabaseConnectorConfig.TABLE_WHITELIST, "DEBEZIUM\\.CUSTOMER")
            .build();

    // Testing.Print.enable();
    int expectedRecordCount = 0;
    connection.execute("INSERT INTO debezium.customer VALUES (1, 'Billie-Bob', 1234.56, TO_DATE('2018/02/22', 'yyyy-mm-dd'))");
    connection.execute("INSERT INTO debezium.customer VALUES (2, 'Bruce', 2345.67, null)");
    connection.execute("COMMIT");
    expectedRecordCount += 2;

    start(OracleConnector.class, config);
    assertConnectorIsRunning();

    SourceRecords records = consumeRecordsByTopic(expectedRecordCount);
    List<SourceRecord> testTableRecords = records.recordsForTopic("server1.DEBEZIUM.CUSTOMER");
    assertThat(testTableRecords).hasSize(expectedRecordCount);

    stopConnector();

    connection.setAutoCommit(false);
    final int OFFSET = 100;
    for (int i = OFFSET; i < expectedRecordCount + OFFSET; i++) {
        connection.executeWithoutCommitting(String.format("INSERT INTO debezium.customer VALUES (%s, 'Brian%s', 2345.67, null)", i, i));
    }
    connection.connection().commit();

    start(OracleConnector.class, config);
    assertConnectorIsRunning();

    assertTxBatch(expectedRecordCount, 100);
    sendTxBatch(expectedRecordCount, 200);
}
 
Example #21
Source File: TestHelper.java    From debezium-incubator with Apache License 2.0 5 votes vote down vote up
private static Configuration.Builder testConfig() {
    JdbcConfiguration jdbcConfiguration = testJdbcConfig();
    Configuration.Builder builder = Configuration.create();

    jdbcConfiguration.forEach(
            (field, value) -> builder.with(OracleConnectorConfig.DATABASE_CONFIG_PREFIX + field, value));

    return builder;
}
 
Example #22
Source File: OracleConnectorIT.java    From debezium-incubator with Apache License 2.0 5 votes vote down vote up
@Test
@FixFor("DBZ-800")
public void shouldReceiveHeartbeatAlsoWhenChangingNonWhitelistedTable() throws Exception {
    TestHelper.dropTable(connection, "debezium.dbz800a");
    TestHelper.dropTable(connection, "debezium.dbz800b");

    // the low heartbeat interval should make sure that a heartbeat message is emitted after each change record
    // received from Postgres
    Configuration config = TestHelper.defaultConfig()
            .with(Heartbeat.HEARTBEAT_INTERVAL, "1")
            .with(OracleConnectorConfig.TABLE_WHITELIST, "DEBEZIUM\\.DBZ800B")
            .build();

    start(OracleConnector.class, config);
    assertConnectorIsRunning();

    waitForSnapshotToBeCompleted(TestHelper.CONNECTOR_NAME, TestHelper.SERVER_NAME);

    connection.execute("CREATE TABLE debezium.dbz800a (id NUMBER(9) NOT NULL, aaa VARCHAR2(100), PRIMARY KEY (id) )");
    connection.execute("CREATE TABLE debezium.dbz800b (id NUMBER(9) NOT NULL, bbb VARCHAR2(100), PRIMARY KEY (id) )");
    connection.execute("INSERT INTO debezium.dbz800a VALUES (1, 'AAA')");
    connection.execute("INSERT INTO debezium.dbz800b VALUES (2, 'BBB')");
    connection.execute("COMMIT");

    // expecting two heartbeat records and one actual change record
    List<SourceRecord> records = consumeRecordsByTopic(3).allRecordsInOrder();

    // expecting no change record for s1.a but a heartbeat
    verifyHeartbeatRecord(records.get(0));

    // and then a change record for s1.b and a heartbeat
    verifyHeartbeatRecord(records.get(1));
    VerifyRecord.isValidInsert(records.get(2), "ID", 2);
}
 
Example #23
Source File: SnapshotDatatypesIT.java    From debezium-incubator with Apache License 2.0 5 votes vote down vote up
@Before
public void before() throws Exception {
    setConsumeTimeout(TestHelper.defaultMessageConsumerPollTimeout(), TimeUnit.SECONDS);
    initializeConnectorTestFramework();
    Testing.Debug.enable();
    Testing.Files.delete(TestHelper.DB_HISTORY_PATH);

    Configuration config = connectorConfig()
            .build();

    start(OracleConnector.class, config);
    assertConnectorIsRunning();

    waitForSnapshotToBeCompleted(TestHelper.CONNECTOR_NAME, TestHelper.SERVER_NAME);
}
 
Example #24
Source File: TransactionMetadataIT.java    From debezium-incubator with Apache License 2.0 5 votes vote down vote up
@Test
public void transactionMetadata() throws Exception {
    Configuration config = TestHelper.defaultConfig()
            .with(RelationalDatabaseConnectorConfig.TABLE_WHITELIST, "DEBEZIUM\\.CUSTOMER")
            .with(OracleConnectorConfig.SNAPSHOT_MODE, SnapshotMode.SCHEMA_ONLY)
            .with(OracleConnectorConfig.PROVIDE_TRANSACTION_METADATA, true)
            .build();

    start(OracleConnector.class, config);
    assertConnectorIsRunning();

    waitForSnapshotToBeCompleted(TestHelper.CONNECTOR_NAME, TestHelper.SERVER_NAME);

    // Testing.Print.enable();
    connection.execute("INSERT INTO debezium.customer VALUES (1, 'Billie-Bob', 1234.56, TO_DATE('2018/02/22', 'yyyy-mm-dd'))");
    connection.execute("COMMIT");

    // TX BEGIN, insert, TX END
    final int expectedRecordCount = 1 + 1 + 1;
    List<SourceRecord> records = consumeRecordsByTopic(expectedRecordCount).allRecordsInOrder();
    assertThat(records).hasSize(expectedRecordCount);

    final String expectedTxId = assertBeginTransaction(records.get(0));

    // insert
    VerifyRecord.isValidInsert(records.get(1), "ID", 1);
    Struct after = (Struct) ((Struct) records.get(1).value()).get("after");
    assertThat(after.get("ID")).isEqualTo(1);
    assertThat(after.get("NAME")).isEqualTo("Billie-Bob");
    assertThat(after.get("SCORE")).isEqualTo(BigDecimal.valueOf(1234.56));
    assertRecordTransactionMetadata(records.get(1), expectedTxId, 1, 1);

    assertEndTransaction(records.get(2), expectedTxId, 1, Collect.hashMapOf("ORCLPDB1.DEBEZIUM.CUSTOMER", 1));
}
 
Example #25
Source File: TestHelper.java    From debezium-incubator with Apache License 2.0 5 votes vote down vote up
private static JdbcConfiguration defaultJdbcConfig() {
    return JdbcConfiguration.copy(Configuration.fromSystemProperties("database."))
            .withDefault(JdbcConfiguration.HOSTNAME, "localhost")
            .withDefault(JdbcConfiguration.PORT, 1521)
            .withDefault(JdbcConfiguration.USER, CONNECTOR_USER)
            .withDefault(JdbcConfiguration.PASSWORD, "xs")
            .withDefault(JdbcConfiguration.DATABASE, "ORCLCDB")
            .build();
}
 
Example #26
Source File: TestHelper.java    From debezium-incubator with Apache License 2.0 5 votes vote down vote up
private static Configuration.Builder adminConfig() {
    JdbcConfiguration jdbcConfiguration = adminJdbcConfig();
    Configuration.Builder builder = Configuration.create();

    jdbcConfiguration.forEach(
            (field, value) -> builder.with(OracleConnectorConfig.DATABASE_CONFIG_PREFIX + field, value));

    return builder;
}
 
Example #27
Source File: TestHelper.java    From debezium-incubator with Apache License 2.0 5 votes vote down vote up
/**
 * Returns a default configuration suitable for most test cases. Can be amended/overridden in individual tests as
 * needed.
 */
public static Configuration.Builder defaultConfig() {
    JdbcConfiguration jdbcConfiguration = defaultJdbcConfig();
    Configuration.Builder builder = Configuration.create();

    jdbcConfiguration.forEach(
            (field, value) -> builder.with(OracleConnectorConfig.DATABASE_CONFIG_PREFIX + field, value));

    return builder.with(RelationalDatabaseConnectorConfig.SERVER_NAME, SERVER_NAME)
            .with(OracleConnectorConfig.PDB_NAME, "ORCLPDB1")
            .with(OracleConnectorConfig.XSTREAM_SERVER_NAME, "dbzxout")
            .with(OracleConnectorConfig.DATABASE_HISTORY, FileDatabaseHistory.class)
            .with(FileDatabaseHistory.FILE_PATH, DB_HISTORY_PATH)
            .with(RelationalDatabaseConnectorConfig.INCLUDE_SCHEMA_CHANGES, false);
}
 
Example #28
Source File: TestHelper.java    From debezium-incubator with Apache License 2.0 5 votes vote down vote up
public static OracleConnection defaultConnection() {
    Configuration config = defaultConfig().build();
    Configuration jdbcConfig = config.subset("database.", true);

    OracleConnection jdbcConnection = new OracleConnection(jdbcConfig, new OracleConnectionFactory());

    String pdbName = new OracleConnectorConfig(config).getPdbName();

    if (pdbName != null) {
        jdbcConnection.setSessionToPdb(pdbName);
    }

    return jdbcConnection;
}
 
Example #29
Source File: TestHelper.java    From debezium-incubator with Apache License 2.0 5 votes vote down vote up
/**
 * Returns a JDBC configuration for the test data schema and user (NOT the XStream user).
 */
private static JdbcConfiguration testJdbcConfig() {
    return JdbcConfiguration.copy(Configuration.fromSystemProperties("database."))
            .withDefault(JdbcConfiguration.HOSTNAME, "localhost")
            .withDefault(JdbcConfiguration.PORT, 1521)
            .withDefault(JdbcConfiguration.USER, "debezium")
            .withDefault(JdbcConfiguration.PASSWORD, "dbz")
            .withDefault(JdbcConfiguration.DATABASE, "ORCLPDB1")
            .build();
}
 
Example #30
Source File: TestHelper.java    From debezium-incubator with Apache License 2.0 5 votes vote down vote up
/**
 * Returns a JDBC configuration for database admin user (NOT the XStream user).
 */
private static JdbcConfiguration adminJdbcConfig() {
    return JdbcConfiguration.copy(Configuration.fromSystemProperties("database.admin."))
            .withDefault(JdbcConfiguration.HOSTNAME, "localhost")
            .withDefault(JdbcConfiguration.PORT, 1521)
            .withDefault(JdbcConfiguration.USER, "sys as sysdba")
            .withDefault(JdbcConfiguration.PASSWORD, "top_secret")
            .withDefault(JdbcConfiguration.DATABASE, "ORCLPDB1")
            .build();
}