com.hazelcast.jet.Job Java Examples

@Test
public void testBatch() {
    Pipeline pipeline = Pipeline.create();
    String query = "Jet flies";
    BatchSource<Status> twitterSearch = TwitterSources.search(credentials, query);
    BatchStage<String> tweets = pipeline
            .readFrom(twitterSearch)
            .map(status -> "@" + status.getUser() + " - " + status.getText());
    tweets.writeTo(AssertionSinks.assertCollectedEventually(60,
            list -> assertGreaterOrEquals("Emits at least 10 tweets in 1 minute.",
                    list.size(), 10)));
    Job job = jet.newJob(pipeline);
    sleepAtLeastSeconds(5);
    try {
        job.join();
        fail("Job should have completed with an AssertionCompletedException, but completed normally");
    } catch (CompletionException e) {
        String errorMsg = e.getCause().getMessage();
        assertTrue("Job was expected to complete with AssertionCompletedException, but completed with: "
                + e.getCause(), errorMsg.contains(AssertionCompletedException.class.getName()));
    }
}
 

private JetPipelineResult run(DAG dag) {
  startClusterIfNeeded(options);

  JetInstance jet =
      getJetInstance(
          options); // todo: we use single client for each job, it might be better to have a
  // shared client with refcount

  Job job = jet.newJob(dag, getJobConfig(options));
  IMap<String, MetricUpdates> metricsAccumulator =
      jet.getMap(JetMetricsContainer.getMetricsMapName(job.getId()));
  JetPipelineResult pipelineResult = new JetPipelineResult(job, metricsAccumulator);
  CompletableFuture<Void> completionFuture =
      job.getFuture()
          .whenCompleteAsync(
              (r, f) -> {
                pipelineResult.freeze(f);
                metricsAccumulator.destroy();
                jet.shutdown();

                stopClusterIfNeeded(options);
              });
  pipelineResult.setCompletionFuture(completionFuture);

  return pipelineResult;
}
 

/**
 * <p>Run one copy of the Moving Average job
 * in this cluster.
 * </p>
 */
@Override
public void run(String... args) throws Exception {
	String prefix = this.getClass().getSimpleName() + " -";
	
	Pipeline pipeline = MovingAverage.build();
   		JobConfig jobConfig = new JobConfig();
   		jobConfig.setName(MyConstants.JOB_NAME);

   		// Run job if not already present
	Job job = this.jetInstance.getJob(jobConfig.getName());
   		if (job == null) {
       	    job = this.jetInstance.newJobIfAbsent(pipeline, jobConfig);
   		}

   		log.info("{} Job '{}', status '{}'.",
           		prefix, job.getName(), job.getStatus());
}
 

public static void main(String[] args) {
    if (FlightDataSource.API_AUTHENTICATION_KEY.equals("YOUR_API_KEY_HERE")) {
         System.err.println("API_AUTHENTICATION_KEY not set in FlightDataSource.java");
         System.exit(1);
    }

    JetInstance jet = getJetInstance();

    Pipeline pipeline = buildPipeline();
    addListener(jet.getMap(TAKE_OFF_MAP), a -> System.out.println("New aircraft taking off: " + a));
    addListener(jet.getMap(LANDING_MAP), a -> System.out.println("New aircraft landing " + a));

    try {
        Job job = jet.newJob(pipeline, new JobConfig().setName("FlightTelemetry").setProcessingGuarantee(ProcessingGuarantee.EXACTLY_ONCE));
        job.join();
    } finally {
        Jet.shutdownAll();
    }
}
 

@Test
public void testStream_withTermFilter() {
    Pipeline pipeline = Pipeline.create();
    List<String> terms = new ArrayList<String>(Arrays.asList("BTC", "ETH"));
    final StreamSource<String> twitterTestStream = TwitterSources.stream(
            credentials, () -> new StatusesFilterEndpoint().trackTerms(terms));
    StreamStage<String> tweets = pipeline
            .readFrom(twitterTestStream)
            .withoutTimestamps()
            .map(rawJson -> Json.parse(rawJson)
                                .asObject()
                                .getString("text", null));

    tweets.writeTo(AssertionSinks.assertCollectedEventually(60,
            list -> assertGreaterOrEquals("Emits at least 20 tweets in 1 min.", list.size(), 20)));
    Job job = jet.newJob(pipeline);
    sleepAtLeastSeconds(5);
    try {
        job.join();
        fail("Job should have completed with an AssertionCompletedException, but completed normally");
    } catch (CompletionException e) {
        String errorMsg = e.getCause().getMessage();
        assertTrue("Job was expected to complete with AssertionCompletedException, but completed with: "
                + e.getCause(), errorMsg.contains(AssertionCompletedException.class.getName()));
    }
}
 

@Test
public void readFromRandomSource() throws Exception {
    System.setProperty("hazelcast.logging.type", "log4j");
    Properties randomProperties = new Properties();
    randomProperties.setProperty("name", "random-source-connector");
    randomProperties.setProperty("connector.class", "sasakitoa.kafka.connect.random.RandomSourceConnector");
    randomProperties.setProperty("generator.class", "sasakitoa.kafka.connect.random.generator.RandomInt");
    randomProperties.setProperty("tasks.max", "1");
    randomProperties.setProperty("messages.per.second", "1000");
    randomProperties.setProperty("topic", "test");
    randomProperties.setProperty("task.summary.enable", "true");

    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(KafkaConnectSources.connect(randomProperties))
            .withoutTimestamps()
            .map(record -> Values.convertToString(record.valueSchema(), record.value()))
            .writeTo(AssertionSinks.assertCollectedEventually(60,
                    list -> assertEquals(ITEM_COUNT, list.size())));

    JobConfig jobConfig = new JobConfig();
    jobConfig.addJar(Objects.requireNonNull(this.getClass()
                                                      .getClassLoader()
                                                      .getResource("random-connector-1.0-SNAPSHOT.jar"))
                                  .getPath()
    );

    Job job = createJetMember().newJob(pipeline, jobConfig);

    sleepAtLeastSeconds(5);

    try {
        job.join();
        fail("Job should have completed with an AssertionCompletedException, but completed normally");
    } catch (CompletionException e) {
        String errorMsg = e.getCause().getMessage();
        assertTrue("Job was expected to complete with AssertionCompletedException, but completed with: "
                + e.getCause(), errorMsg.contains(AssertionCompletedException.class.getName()));
    }
}
 

@Test
public void when_readFromPulsarConsumer_then_jobGetsAllPublishedMessages() {
    JetInstance[] instances = new JetInstance[2];
    Arrays.setAll(instances, i -> createJetMember());

    String topicName = randomName();
    StreamSource<String> pulsarConsumerSrc = setupConsumerSource(topicName,
            x -> new String(x.getData(), StandardCharsets.UTF_8));

    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(pulsarConsumerSrc)
            .withoutTimestamps()
            .writeTo(AssertionSinks.assertCollectedEventually(60,
                    list -> {
                        assertEquals("# of Emitted items should be equal to # of published items",
                                ITEM_COUNT, list.size());
                        for (int i = 0; i < ITEM_COUNT; i++) {
                            String message = "hello-pulsar-" + i;
                            Assert.assertTrue("missing entry: " + message, list.contains(message));
                        }
                    })
            );
    Job job = instances[0].newJob(pipeline);
    assertJobStatusEventually(job, JobStatus.RUNNING);

    produceMessages("hello-pulsar", topicName, ITEM_COUNT);

    try {
        job.join();
        fail("Job should have completed with an AssertionCompletedException, but completed normally");
    } catch (CompletionException e) {
        String errorMsg = e.getCause().getMessage();
        assertTrue("Job was expected to complete with AssertionCompletedException, but completed with: "
                + e.getCause(), errorMsg.contains(AssertionCompletedException.class.getName()));
    }
    for (JetInstance instance:instances) {
        instance.shutdown();
    }
}
 

public static void main(String[] args) {
    Pipeline p = buildPipeline();

    JetInstance jet = Jet.bootstrappedInstance();

    try {
        Job job = jet.newJob(p);
        job.join();
    } finally {
        jet.shutdown();
    }
}
 

public static void main(String[] args) {
    Pipeline p = buildPipeline();

    JetInstance jet = Jet.bootstrappedInstance();

    try {
        Job job = jet.newJob(p);
        job.join();
    } finally {
        jet.shutdown();
    }
}
 

public static void main (String[] args) {
    Pipeline p = buildPipeline();

    JetInstance jet = Jet.bootstrappedInstance();

    try {
        Job job = jet.newJob(p);
        job.join();
    } finally {
        jet.shutdown();
    }
}
 

public static void main(String[] args) {
    Pipeline p = buildPipeline();

    JetInstance jet = Jet.bootstrappedInstance();

    try {
        Job job = jet.newJob(p);
        job.join();
    } finally {
        jet.shutdown();
    }
}
 

public static void main(String[] args) {
    if (args.length != 2) {
        System.err.println("Missing command-line arguments: <model-file-path> <validation-input-data>");
        System.exit(1);
    }

    Path modelFile = Paths.get(args[0]).toAbsolutePath();
    Path inputFile = Paths.get(args[1]).toAbsolutePath();
    validateFileReadable(modelFile);
    validateFileReadable(inputFile);

    System.setProperty("hazelcast.logging.type", "log4j");

    JetInstance jet = Jet.newJetInstance();

    JobConfig jobConfig = new JobConfig();
    jobConfig.setName("h2o Breast Cancer Classification");
    jobConfig.attachFile(modelFile.toString(), "model");

    Job job = jet.newJob(buildPipeline(inputFile), jobConfig);

    try {
        job.join();
    } finally {
        jet.shutdown();
    }
}
 

@Test
public void stream() {
    int addCount = 500;
    int streamCount = 2;

    for (int i = 0; i < streamCount; i++) {
        fillStream("stream-" + i, addCount);
    }

    Map<String, String> streamOffsets = new HashMap<>();
    for (int i = 0; i < streamCount; i++) {
        streamOffsets.put("stream-" + i, "0");
    }

    Sink<Object> sink = SinkBuilder
            .sinkBuilder("set", c -> c.jetInstance().getHazelcastInstance().getSet("set"))
            .receiveFn(Set::add)
            .build();

    Pipeline p = Pipeline.create();
    p.readFrom(RedisSources.stream("source", uri, streamOffsets,
            mes -> mes.getStream() + " - " + mes.getId()))
            .withoutTimestamps()
            .writeTo(sink);

    Job job = instance.newJob(p);

    Collection<Object> set = instance.getHazelcastInstance().getSet("set");
    assertTrueEventually(() -> assertEquals(addCount * streamCount, set.size()));

    job.cancel();
}
 

@Test
public void when_projectionFunctionProvided_thenAppliedToReadRecords() {
    String topicName = randomName();
    // Add a suffix to messages so that this projectionFn does a bit more than byte->String conversion.
    StreamSource<String> pulsarConsumerSrc = setupConsumerSource(topicName,
            x -> new String(x.getData(), StandardCharsets.UTF_8) + "-suffix");

    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(pulsarConsumerSrc)
            .withoutTimestamps()
            .writeTo(AssertionSinks.assertCollectedEventually(60,
                    list -> {
                        assertEquals("# of Emitted items should be equal to # of published items",
                                ITEM_COUNT, list.size());
                        for (int i = 0; i < ITEM_COUNT; i++) {
                            String message = "hello-pulsar-" + i + "-suffix";
                            Assert.assertTrue("missing entry: " + message, list.contains(message));
                        }
                    })
            );
    Job job = createJetMember().newJob(pipeline);
    assertJobStatusEventually(job, JobStatus.RUNNING);

    produceMessages("hello-pulsar", topicName, ITEM_COUNT);

    try {
        job.join();
        fail("Job should have completed with an AssertionCompletedException, but completed normally");
    } catch (CompletionException e) {
        String errorMsg = e.getCause().getMessage();
        assertTrue("Job was expected to complete with AssertionCompletedException, but completed with: "
                + e.getCause(), errorMsg.contains(AssertionCompletedException.class.getName()));
    }
}
 

@Test
public void testTimestampedStream_userFilter() {
    Pipeline pipeline = Pipeline.create();
    List<Long> userIds = new ArrayList<Long>(
            Arrays.asList(612473L, 759251L, 1367531L, 34713362L, 51241574L, 87818409L));
    final StreamSource<String> twitterTestStream = TwitterSources.timestampedStream(
            credentials, () -> new StatusesFilterEndpoint().followings(userIds));
    StreamStage<String> tweets = pipeline
            .readFrom(twitterTestStream)
            .withNativeTimestamps(1)
            .map(rawJson -> Json.parse(rawJson)
                                .asObject()
                                .getString("text", null));
    tweets.writeTo(AssertionSinks.assertCollectedEventually(60,
            list -> assertGreaterOrEquals("Emits at least 15 tweets in 1 min.",
                    list.size(), 15)));
    Job job = jet.newJob(pipeline);
    sleepAtLeastSeconds(5);
    try {
        job.join();
        fail("Job should have completed with an AssertionCompletedException, but completed normally");
    } catch (CompletionException e) {
        String errorMsg = e.getCause().getMessage();
        assertTrue("Job was expected to complete with AssertionCompletedException, but completed with: "
                + e.getCause(), errorMsg.contains(AssertionCompletedException.class.getName()));
    }
}
 

@Test
public void testTimestampedStream_termFilter() {
    Pipeline pipeline = Pipeline.create();
    List<String> terms = new ArrayList<String>(Arrays.asList("San Mateo", "Brno", "London", "Istanbul"));

    final StreamSource<String> twitterTestStream = TwitterSources.timestampedStream(
            credentials, () -> new StatusesFilterEndpoint().trackTerms(terms));
    StreamStage<String> tweets = pipeline
            .readFrom(twitterTestStream)
            .withNativeTimestamps(0)
            .map(rawJson -> Json.parse(rawJson)
                                .asObject()
                                .getString("text", null));
    tweets.writeTo(AssertionSinks.assertCollectedEventually(60,
            list -> assertGreaterOrEquals("Emits at least 20 tweets in 1 min.",
                    list.size(), 20)));
    Job job = jet.newJob(pipeline);
    sleepAtLeastSeconds(5);
    try {
        job.join();
        fail("Job should have completed with an AssertionCompletedException, but completed normally");
    } catch (CompletionException e) {
        String errorMsg = e.getCause().getMessage();
        assertTrue("Job was expected to complete with AssertionCompletedException, but completed with: "
                + e.getCause(), errorMsg.contains(AssertionCompletedException.class.getName()));
    }
}
 

@Test
public void testStream_userFilter() {
    Pipeline pipeline = Pipeline.create();
    List<Long> userIds = new ArrayList<Long>(
            Arrays.asList(612473L, 759251L, 1367531L, 34713362L, 51241574L, 87818409L));
    final StreamSource<String> twitterTestStream = TwitterSources.stream(credentials,
            () -> new StatusesFilterEndpoint().followings(userIds));
    StreamStage<String> tweets = pipeline
            .readFrom(twitterTestStream)
            .withoutTimestamps()
            .map(rawJson -> Json.parse(rawJson)
                                .asObject()
                                .getString("text", null));
    tweets.writeTo(AssertionSinks.assertCollectedEventually(60,
            list -> assertGreaterOrEquals("Emits at least 15 tweets in 1 min.",
                    list.size(), 15)));
    Job job = jet.newJob(pipeline);
    sleepAtLeastSeconds(5);
    try {
        job.join();
        fail("Job should have completed with an AssertionCompletedException, but completed normally");
    } catch (CompletionException e) {
        String errorMsg = e.getCause().getMessage();
        assertTrue("Job was expected to complete with AssertionCompletedException, but completed with: "
                + e.getCause(), errorMsg.contains(AssertionCompletedException.class.getName()));
    }
}
 

@Test
public void testBatchMock() {
    String baseUrl = "http://" + server.getHostName() + ":" + server.getPort() + "/";
    System.setProperty("twitter4j.restBaseURL", baseUrl);

    Pipeline pipeline = Pipeline.create();
    String query = "Jet flies";
    BatchSource<Status> twitterSearch = TwitterSources.search(
            getCredentials(), query);
    BatchStage<String> tweets = pipeline
            .readFrom(twitterSearch)
            .map(status -> "@" + status.getUser().getName() + " - " + status.getText());

    tweets.writeTo(AssertionSinks.assertCollectedEventually(10,
            list -> assertGreaterOrEquals("Emits at least 30 tweets in 10 secs.",
                    list.size(), 30)));
    Job job = createJetMember().newJob(pipeline);

    sleepAtLeastSeconds(2);

    try {
        job.join();
        fail("Job should have completed with an AssertionCompletedException, but completed normally");
    } catch (CompletionException e) {
        String errorMsg = e.getCause().getMessage();
        assertTrue("Job was expected to complete with AssertionCompletedException, but completed with: "
                + e.getCause(), errorMsg.contains(AssertionCompletedException.class.getName()));
    }
}
 

@Test
public void streamApiMockTest() {

    List<String> terms = new ArrayList<>(Arrays.asList("San Mateo", "Brno", "London", "Istanbul"));
    final StreamSource<String> twitterTestStream = TwitterSources.timestampedStream(getCredentials(),
            "http://" + server.getHostName() + ":" + server.getPort(),
            () -> new StatusesFilterEndpoint().trackTerms(terms));

    Pipeline pipeline = Pipeline.create();
    StreamStage<String> tweets = pipeline
            .readFrom(twitterTestStream)
            .withNativeTimestamps(0)
            .map(rawJson -> Json.parse(rawJson)
                                .asObject()
                                .getString("text", null));
    tweets.writeTo(AssertionSinks.assertCollectedEventually(10,
            list -> assertGreaterOrEquals("Emits at least 100 tweets in 1 min.",
                    list.size(), 100)));
    Job job = createJetMember().newJob(pipeline);

    sleepAtLeastSeconds(5);

    try {
        job.join();
        fail("Job should have completed with an AssertionCompletedException, but completed normally");
    } catch (CompletionException e) {
        String errorMsg = e.getCause().getMessage();
        assertTrue("Job was expected to complete with AssertionCompletedException, but completed with: "
                + e.getCause(), errorMsg.contains(AssertionCompletedException.class.getName()));
    }
}
 

@Test
public void readFromMySql() throws Exception {
    // given
    Configuration configuration = Configuration
            .create()
            .with("name", "mysql-inventory-connector")
            .with("connector.class", "io.debezium.connector.mysql.MySqlConnector")
            /* begin connector properties */
            .with("database.hostname", mysql.getContainerIpAddress())
            .with("database.port", mysql.getMappedPort(MYSQL_PORT))
            .with("database.user", "debezium")
            .with("database.password", "dbz")
            .with("database.server.id", "184054")
            .with("database.server.name", "dbserver1")
            .with("database.whitelist", "inventory")
            .with("table.whitelist", "inventory.customers")
            .with("include.schema.changes", "false")
            .with("database.history.hazelcast.list.name", "test")
            .build();

    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(DebeziumSources.cdc(configuration))
            .withoutTimestamps()
            .map(record -> Values.convertToString(record.valueSchema(), record.value()))
            .filterUsingService(ServiceFactories.sharedService(context -> {
                Serde<EventRecord> serde = DebeziumSerdes.payloadJson(EventRecord.class);
                serde.configure(Collections.emptyMap(), false);
                return serde.deserializer();
            }, Deserializer::close), (deserializer, record) -> {
                EventRecord eventRecord = deserializer.deserialize("", record.getBytes());
                return eventRecord.isUpdate();
            })
            .writeTo(AssertionSinks.assertCollectedEventually(30,
                    list -> Assert.assertTrue(list.stream().anyMatch(s -> s.contains("Anne Marie")))));

    JobConfig jobConfig = new JobConfig();
    jobConfig.addJarsInZip(Objects.requireNonNull(this.getClass()
                                                      .getClassLoader()
                                                      .getResource("debezium-connector-mysql.zip")));

    // when
    JetInstance jet = createJetMember();
    Job job = jet.newJob(pipeline, jobConfig);
    assertJobStatusEventually(job, JobStatus.RUNNING);

    sleepAtLeastSeconds(10);
    // update a record
    try (Connection connection = DriverManager.getConnection(mysql.withDatabaseName("inventory").getJdbcUrl(),
            mysql.getUsername(), mysql.getPassword())) {
        PreparedStatement preparedStatement = connection
                .prepareStatement("UPDATE customers SET first_name='Anne Marie' WHERE id=1004;");
        preparedStatement.executeUpdate();
    }


    // then
    try {
        job.join();
        Assert.fail("Job should have completed with an AssertionCompletedException, but completed normally");
    } catch (CompletionException e) {
        String errorMsg = e.getCause().getMessage();
        Assert.assertTrue("Job was expected to complete with " +
                        "AssertionCompletedException, but completed with: " + e.getCause(),
                errorMsg.contains(AssertionCompletedException.class.getName()));
    }
}
 

@Test
public void readFromPostgres() throws Exception {
    Configuration configuration = Configuration
            .create()
            .with("name", "postgres-inventory-connector")
            .with("connector.class", "io.debezium.connector.postgresql.PostgresConnector")
            /* begin connector properties */
            .with("tasks.max", "1")
            .with("database.hostname", postgres.getContainerIpAddress())
            .with("database.port", postgres.getMappedPort(POSTGRESQL_PORT))
            .with("database.user", "postgres")
            .with("database.password", "postgres")
            .with("database.dbname", "postgres")
            .with("database.server.name", "dbserver1")
            .with("schema.whitelist", "inventory")
            .with("database.history.hazelcast.list.name", "test")
            .build();

    JetInstance jet = createJetMember();

    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(DebeziumSources.cdc(configuration))
            .withoutTimestamps()
            .map(record -> Values.convertToString(record.valueSchema(), record.value()))
            .writeTo(AssertionSinks.assertCollectedEventually(60,
                    list -> assertTrue(list.stream().anyMatch(s -> s.contains("Anne Marie")))));

    JobConfig jobConfig = new JobConfig();
    jobConfig.addJarsInZip(Objects.requireNonNull(this.getClass()
                                                      .getClassLoader()
                                                      .getResource("debezium-connector-postgres.zip")));

    Job job = jet.newJob(pipeline, jobConfig);
    assertJobStatusEventually(job, JobStatus.RUNNING);

    // update record
    try (Connection connection = DriverManager.getConnection(postgres.getJdbcUrl(), postgres.getUsername(),
            postgres.getPassword())) {
        connection.setSchema("inventory");
        PreparedStatement preparedStatement = connection.prepareStatement("update customers set " +
                "first_name = 'Anne Marie' where id = 1004;");
        preparedStatement.executeUpdate();
    }

    try {
        job.join();
        fail("Job should have completed with an AssertionCompletedException, but completed normally");
    } catch (CompletionException e) {
        String errorMsg = e.getCause().getMessage();
        assertTrue("Job was expected to complete with AssertionCompletedException, but completed with: "
                + e.getCause(), errorMsg.contains(AssertionCompletedException.class.getName()));
    }


}
 

@Test
public void readFromMsSql() throws Exception {
    execInContainer("setup.sql");
    assertTrueEventually(() -> {
        Container.ExecResult result = execInContainer("cdc.sql");
        assertContains(result.getStdout(), "already");
    });

    Configuration configuration = Configuration
            .create()
            .with("name", "mssql-inventory-connector")
            .with("connector.class", "io.debezium.connector.sqlserver.SqlServerConnector")
            /* begin connector properties */
            .with("tasks.max", "1")
            .with("database.hostname", mssql.getContainerIpAddress())
            .with("database.port", mssql.getMappedPort(MS_SQL_SERVER_PORT))
            .with("database.user", mssql.getUsername())
            .with("database.password", mssql.getPassword())
            .with("database.dbname", "MyDB")
            .with("database.server.name", "fulfillment")
            .with("table.whitelist", "inventory.customers")
            .with("database.history.hazelcast.list.name", "test")
            .build();

    JetInstance jet = createJetMember();

    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(DebeziumSources.cdc(configuration))
            .withoutTimestamps()
            .map(record -> Values.convertToString(record.valueSchema(), record.value()))
            .writeTo(AssertionSinks.assertCollectedEventually(60,
                    list -> assertTrue(list.stream().anyMatch(s -> s.contains("Anne Marie")))));

    JobConfig jobConfig = new JobConfig();
    jobConfig.addJarsInZip(Objects.requireNonNull(this.getClass()
                                                      .getClassLoader()
                                                      .getResource("debezium-connector-sqlserver.zip")));

    Job job = jet.newJob(pipeline, jobConfig);
    assertJobStatusEventually(job, JobStatus.RUNNING);

    sleepAtLeastSeconds(30);
    // update record
    try (Connection connection = DriverManager.getConnection(mssql.getJdbcUrl() + ";databaseName=MyDB",
            mssql.getUsername(), mssql.getPassword())) {
        connection.setSchema("inventory");
        PreparedStatement preparedStatement = connection.prepareStatement("update MyDB.inventory.customers set " +
                "first_name = 'Anne Marie' where id = 1001;");
        preparedStatement.executeUpdate();
    }

    try {
        job.join();
        fail("Job should have completed with an AssertionCompletedException, but completed normally");
    } catch (CompletionException e) {
        String errorMsg = e.getCause().getMessage();
        assertTrue("Job was expected to complete with AssertionCompletedException, but completed with: "
                + e.getCause(), errorMsg.contains(AssertionCompletedException.class.getName()));
    }


}
 

JetPipelineResult(@Nonnull Job job, @Nonnull IMap<String, MetricUpdates> metricsAccumulator) {
  this.job = Objects.requireNonNull(job);
  // save the terminal state when the job completes because the `job` instance will become invalid
  // afterwards
  metricResults = new JetMetricResults(metricsAccumulator);
}
 

@Test
public void readFromMongoDb() throws Exception {
    // populate initial data
    mongo.execInContainer("sh", "-c", "/usr/local/bin/init-inventory.sh");

    Configuration configuration = Configuration
            .create()
            .with("name", "mongodb-inventory-connector")
            .with("connector.class", "io.debezium.connector.mongodb.MongoDbConnector")
            /* begin connector properties */
            .with("mongodb.hosts", "rs0/" + mongo.getContainerIpAddress() + ":"
                    + mongo.getMappedPort(MongoDBContainer.MONGODB_PORT))
            .with("mongodb.name", "fullfillment")
            .with("mongodb.user", "debezium")
            .with("mongodb.password", "dbz")
            .with("mongodb.members.auto.discover", "false")
            .with("collection.whitelist", "inventory.*")
            .with("database.history.hazelcast.list.name", "test")
            .build();

    JetInstance jet = createJetMember();

    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(DebeziumSources.cdc(configuration))
            .withoutTimestamps()
            .map(record -> Values.convertToString(record.valueSchema(), record.value()))
            .writeTo(AssertionSinks.assertCollectedEventually(60,
                    list -> Assert.assertTrue(list.stream().anyMatch(s -> s.contains("Jason")))));

    JobConfig jobConfig = new JobConfig();
    jobConfig.addJarsInZip(Objects.requireNonNull(this.getClass()
                                                      .getClassLoader()
                                                      .getResource("debezium-connector-mongodb.zip")));

    Job job = jet.newJob(pipeline, jobConfig);
    assertJobStatusEventually(job, JobStatus.RUNNING);

    // update record
    mongo.execInContainer("sh", "-c", "/usr/local/bin/insertData.sh");

    try {
        job.join();
        Assert.fail("Job should have completed with an AssertionCompletedException, but completed normally");
    } catch (CompletionException e) {
        String errorMsg = e.getCause().getMessage();
        Assert.assertTrue("Job was expected to complete with AssertionCompletedException, " +
                "but completed with: " + e.getCause(), errorMsg.contains(AssertionCompletedException.class.getName()));
    }


}
 

@Test
public void testStream_whenWatchDatabase() {
    IList<Document> list = jet.getList("list");

    String connectionString = mongoContainer.connectionString();
    long value = startAtOperationTime.getValue();

    StreamSource<? extends Document> source = MongoDBSourceBuilder
            .streamDatabase(SOURCE_NAME, () -> MongoClients.create(connectionString))
            .databaseFn(client -> client.getDatabase(DB_NAME))
            .destroyFn(MongoClient::close)
            .searchFn(db -> {
                List<Bson> aggregates = new ArrayList<>();
                aggregates.add(Aggregates.match(new Document("fullDocument.val", new Document("$gte", 10))
                        .append("operationType", "insert")));

                aggregates.add(Aggregates.project(new Document("fullDocument.val", 1).append("_id", 1)));
                return db.watch(aggregates);
            })
            .mapFn(ChangeStreamDocument::getFullDocument)
            .startAtOperationTimeFn(client -> new BsonTimestamp(value))
            .build();


    Pipeline p = Pipeline.create();
    p.readFrom(source)
     .withNativeTimestamps(0)
     .writeTo(Sinks.list(list));

    Job job = jet.newJob(p);

    MongoCollection<Document> col1 = collection("col1");
    MongoCollection<Document> col2 = collection("col2");

    col1.insertOne(new Document("val", 1));
    col1.insertOne(new Document("val", 10).append("foo", "bar"));

    col2.insertOne(new Document("val", 2));
    col2.insertOne(new Document("val", 11).append("foo", "bar"));

    assertTrueEventually(() -> {
        assertEquals(2, list.size());
        list.forEach(document -> assertNull(document.get("foo")));

        assertEquals(10, list.get(0).get("val"));
        assertEquals(11, list.get(1).get("val"));

    });

    col1.insertOne(new Document("val", 3));
    col1.insertOne(new Document("val", 12).append("foo", "bar"));

    col2.insertOne(new Document("val", 4));
    col2.insertOne(new Document("val", 13).append("foo", "bar"));

    assertTrueEventually(() -> {
        assertEquals(4, list.size());
        list.forEach(document -> assertNull(document.get("foo")));

        assertEquals(12, list.get(2).get("val"));
        assertEquals(13, list.get(3).get("val"));
    });

    job.cancel();

}
 

@Test
public void testStream_whenWatchAll() {
    IList<Document> list = jet.getList("list");

    String connectionString = mongoContainer.connectionString();
    long value = startAtOperationTime.getValue();

    StreamSource<? extends Document> source = MongoDBSourceBuilder
            .streamAll(SOURCE_NAME, () -> MongoClients.create(connectionString))
            .destroyFn(MongoClient::close)
            .searchFn(client -> {
                List<Bson> aggregates = new ArrayList<>();
                aggregates.add(Aggregates.match(new Document("fullDocument.val", new Document("$gt", 10))
                        .append("operationType", "insert")));

                aggregates.add(Aggregates.project(new Document("fullDocument.val", 1).append("_id", 1)));
                return client.watch(aggregates);
            })
            .mapFn(ChangeStreamDocument::getFullDocument)
            .startAtOperationTimeFn(client -> new BsonTimestamp(value))
            .build();

    Pipeline p = Pipeline.create();
    p.readFrom(source)
     .withNativeTimestamps(0)
     .writeTo(Sinks.list(list));

    Job job = jet.newJob(p);

    MongoCollection<Document> col1 = collection("db1", "col1");
    MongoCollection<Document> col2 = collection("db1", "col2");
    MongoCollection<Document> col3 = collection("db2", "col3");

    col1.insertOne(new Document("val", 1));
    col1.insertOne(new Document("val", 11).append("foo", "bar"));
    col2.insertOne(new Document("val", 2));
    col2.insertOne(new Document("val", 12).append("foo", "bar"));
    col3.insertOne(new Document("val", 3));
    col3.insertOne(new Document("val", 13).append("foo", "bar"));

    assertTrueEventually(() -> {
        assertEquals(3, list.size());
        list.forEach(document -> assertNull(document.get("foo")));

        assertEquals(11, list.get(0).get("val"));
        assertEquals(12, list.get(1).get("val"));
        assertEquals(13, list.get(2).get("val"));
    });

    col1.insertOne(new Document("val", 4));
    col1.insertOne(new Document("val", 14).append("foo", "bar"));
    col2.insertOne(new Document("val", 5));
    col2.insertOne(new Document("val", 15).append("foo", "bar"));
    col2.insertOne(new Document("val", 6));
    col2.insertOne(new Document("val", 16).append("foo", "bar"));

    assertTrueEventually(() -> {
        assertEquals(6, list.size());
        list.forEach(document -> assertNull(document.get("foo")));

        assertEquals(14, list.get(3).get("val"));
        assertEquals(15, list.get(4).get("val"));
        assertEquals(16, list.get(5).get("val"));
    });

    job.cancel();

}