Java Code Examples for com.hazelcast.jet.JetInstance#newJob()

public static void main(String[] args) {
    if (FlightDataSource.API_AUTHENTICATION_KEY.equals("YOUR_API_KEY_HERE")) {
         System.err.println("API_AUTHENTICATION_KEY not set in FlightDataSource.java");
         System.exit(1);
    }

    JetInstance jet = getJetInstance();

    Pipeline pipeline = buildPipeline();
    addListener(jet.getMap(TAKE_OFF_MAP), a -> System.out.println("New aircraft taking off: " + a));
    addListener(jet.getMap(LANDING_MAP), a -> System.out.println("New aircraft landing " + a));

    try {
        Job job = jet.newJob(pipeline, new JobConfig().setName("FlightTelemetry").setProcessingGuarantee(ProcessingGuarantee.EXACTLY_ONCE));
        job.join();
    } finally {
        Jet.shutdownAll();
    }
}
 

private JetPipelineResult run(DAG dag) {
  startClusterIfNeeded(options);

  JetInstance jet =
      getJetInstance(
          options); // todo: we use single client for each job, it might be better to have a
  // shared client with refcount

  Job job = jet.newJob(dag, getJobConfig(options));
  IMap<String, MetricUpdates> metricsAccumulator =
      jet.getMap(JetMetricsContainer.getMetricsMapName(job.getId()));
  JetPipelineResult pipelineResult = new JetPipelineResult(job, metricsAccumulator);
  CompletableFuture<Void> completionFuture =
      job.getFuture()
          .whenCompleteAsync(
              (r, f) -> {
                pipelineResult.freeze(f);
                metricsAccumulator.destroy();
                jet.shutdown();

                stopClusterIfNeeded(options);
              });
  pipelineResult.setCompletionFuture(completionFuture);

  return pipelineResult;
}
 

public static void main(String[] args) {
    if (args.length != 2) {
        System.err.println("Missing command-line arguments: <model-file-path> <validation-input-data>");
        System.exit(1);
    }

    Path modelFile = Paths.get(args[0]).toAbsolutePath();
    Path inputFile = Paths.get(args[1]).toAbsolutePath();
    validateFileReadable(modelFile);
    validateFileReadable(inputFile);

    System.setProperty("hazelcast.logging.type", "log4j");

    JetInstance jet = Jet.newJetInstance();

    JobConfig jobConfig = new JobConfig();
    jobConfig.setName("h2o Breast Cancer Classification");
    jobConfig.attachFile(modelFile.toString(), "model");

    Job job = jet.newJob(buildPipeline(inputFile), jobConfig);

    try {
        job.join();
    } finally {
        jet.shutdown();
    }
}
 

public static void main(String[] args) {
    Pipeline p = buildPipeline();

    JetInstance jet = Jet.bootstrappedInstance();

    try {
        Job job = jet.newJob(p);
        job.join();
    } finally {
        jet.shutdown();
    }
}
 

public static void main (String[] args) {
    Pipeline p = buildPipeline();

    JetInstance jet = Jet.bootstrappedInstance();

    try {
        Job job = jet.newJob(p);
        job.join();
    } finally {
        jet.shutdown();
    }
}
 

public static void main(String[] args) {
    Pipeline p = buildPipeline();

    JetInstance jet = Jet.bootstrappedInstance();

    try {
        Job job = jet.newJob(p);
        job.join();
    } finally {
        jet.shutdown();
    }
}
 

public static void main(String[] args) {
    Pipeline p = buildPipeline();

    JetInstance jet = Jet.bootstrappedInstance();

    try {
        Job job = jet.newJob(p);
        job.join();
    } finally {
        jet.shutdown();
    }
}
 

@Test
public void readFromPostgres() throws Exception {
    Configuration configuration = Configuration
            .create()
            .with("name", "postgres-inventory-connector")
            .with("connector.class", "io.debezium.connector.postgresql.PostgresConnector")
            /* begin connector properties */
            .with("tasks.max", "1")
            .with("database.hostname", postgres.getContainerIpAddress())
            .with("database.port", postgres.getMappedPort(POSTGRESQL_PORT))
            .with("database.user", "postgres")
            .with("database.password", "postgres")
            .with("database.dbname", "postgres")
            .with("database.server.name", "dbserver1")
            .with("schema.whitelist", "inventory")
            .with("database.history.hazelcast.list.name", "test")
            .build();

    JetInstance jet = createJetMember();

    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(DebeziumSources.cdc(configuration))
            .withoutTimestamps()
            .map(record -> Values.convertToString(record.valueSchema(), record.value()))
            .writeTo(AssertionSinks.assertCollectedEventually(60,
                    list -> assertTrue(list.stream().anyMatch(s -> s.contains("Anne Marie")))));

    JobConfig jobConfig = new JobConfig();
    jobConfig.addJarsInZip(Objects.requireNonNull(this.getClass()
                                                      .getClassLoader()
                                                      .getResource("debezium-connector-postgres.zip")));

    Job job = jet.newJob(pipeline, jobConfig);
    assertJobStatusEventually(job, JobStatus.RUNNING);

    // update record
    try (Connection connection = DriverManager.getConnection(postgres.getJdbcUrl(), postgres.getUsername(),
            postgres.getPassword())) {
        connection.setSchema("inventory");
        PreparedStatement preparedStatement = connection.prepareStatement("update customers set " +
                "first_name = 'Anne Marie' where id = 1004;");
        preparedStatement.executeUpdate();
    }

    try {
        job.join();
        fail("Job should have completed with an AssertionCompletedException, but completed normally");
    } catch (CompletionException e) {
        String errorMsg = e.getCause().getMessage();
        assertTrue("Job was expected to complete with AssertionCompletedException, but completed with: "
                + e.getCause(), errorMsg.contains(AssertionCompletedException.class.getName()));
    }


}
 

@Test
public void readFromMySql() throws Exception {
    // given
    Configuration configuration = Configuration
            .create()
            .with("name", "mysql-inventory-connector")
            .with("connector.class", "io.debezium.connector.mysql.MySqlConnector")
            /* begin connector properties */
            .with("database.hostname", mysql.getContainerIpAddress())
            .with("database.port", mysql.getMappedPort(MYSQL_PORT))
            .with("database.user", "debezium")
            .with("database.password", "dbz")
            .with("database.server.id", "184054")
            .with("database.server.name", "dbserver1")
            .with("database.whitelist", "inventory")
            .with("table.whitelist", "inventory.customers")
            .with("include.schema.changes", "false")
            .with("database.history.hazelcast.list.name", "test")
            .build();

    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(DebeziumSources.cdc(configuration))
            .withoutTimestamps()
            .map(record -> Values.convertToString(record.valueSchema(), record.value()))
            .filterUsingService(ServiceFactories.sharedService(context -> {
                Serde<EventRecord> serde = DebeziumSerdes.payloadJson(EventRecord.class);
                serde.configure(Collections.emptyMap(), false);
                return serde.deserializer();
            }, Deserializer::close), (deserializer, record) -> {
                EventRecord eventRecord = deserializer.deserialize("", record.getBytes());
                return eventRecord.isUpdate();
            })
            .writeTo(AssertionSinks.assertCollectedEventually(30,
                    list -> Assert.assertTrue(list.stream().anyMatch(s -> s.contains("Anne Marie")))));

    JobConfig jobConfig = new JobConfig();
    jobConfig.addJarsInZip(Objects.requireNonNull(this.getClass()
                                                      .getClassLoader()
                                                      .getResource("debezium-connector-mysql.zip")));

    // when
    JetInstance jet = createJetMember();
    Job job = jet.newJob(pipeline, jobConfig);
    assertJobStatusEventually(job, JobStatus.RUNNING);

    sleepAtLeastSeconds(10);
    // update a record
    try (Connection connection = DriverManager.getConnection(mysql.withDatabaseName("inventory").getJdbcUrl(),
            mysql.getUsername(), mysql.getPassword())) {
        PreparedStatement preparedStatement = connection
                .prepareStatement("UPDATE customers SET first_name='Anne Marie' WHERE id=1004;");
        preparedStatement.executeUpdate();
    }


    // then
    try {
        job.join();
        Assert.fail("Job should have completed with an AssertionCompletedException, but completed normally");
    } catch (CompletionException e) {
        String errorMsg = e.getCause().getMessage();
        Assert.assertTrue("Job was expected to complete with " +
                        "AssertionCompletedException, but completed with: " + e.getCause(),
                errorMsg.contains(AssertionCompletedException.class.getName()));
    }
}
 

@Test
public void readFromMsSql() throws Exception {
    execInContainer("setup.sql");
    assertTrueEventually(() -> {
        Container.ExecResult result = execInContainer("cdc.sql");
        assertContains(result.getStdout(), "already");
    });

    Configuration configuration = Configuration
            .create()
            .with("name", "mssql-inventory-connector")
            .with("connector.class", "io.debezium.connector.sqlserver.SqlServerConnector")
            /* begin connector properties */
            .with("tasks.max", "1")
            .with("database.hostname", mssql.getContainerIpAddress())
            .with("database.port", mssql.getMappedPort(MS_SQL_SERVER_PORT))
            .with("database.user", mssql.getUsername())
            .with("database.password", mssql.getPassword())
            .with("database.dbname", "MyDB")
            .with("database.server.name", "fulfillment")
            .with("table.whitelist", "inventory.customers")
            .with("database.history.hazelcast.list.name", "test")
            .build();

    JetInstance jet = createJetMember();

    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(DebeziumSources.cdc(configuration))
            .withoutTimestamps()
            .map(record -> Values.convertToString(record.valueSchema(), record.value()))
            .writeTo(AssertionSinks.assertCollectedEventually(60,
                    list -> assertTrue(list.stream().anyMatch(s -> s.contains("Anne Marie")))));

    JobConfig jobConfig = new JobConfig();
    jobConfig.addJarsInZip(Objects.requireNonNull(this.getClass()
                                                      .getClassLoader()
                                                      .getResource("debezium-connector-sqlserver.zip")));

    Job job = jet.newJob(pipeline, jobConfig);
    assertJobStatusEventually(job, JobStatus.RUNNING);

    sleepAtLeastSeconds(30);
    // update record
    try (Connection connection = DriverManager.getConnection(mssql.getJdbcUrl() + ";databaseName=MyDB",
            mssql.getUsername(), mssql.getPassword())) {
        connection.setSchema("inventory");
        PreparedStatement preparedStatement = connection.prepareStatement("update MyDB.inventory.customers set " +
                "first_name = 'Anne Marie' where id = 1001;");
        preparedStatement.executeUpdate();
    }

    try {
        job.join();
        fail("Job should have completed with an AssertionCompletedException, but completed normally");
    } catch (CompletionException e) {
        String errorMsg = e.getCause().getMessage();
        assertTrue("Job was expected to complete with AssertionCompletedException, but completed with: "
                + e.getCause(), errorMsg.contains(AssertionCompletedException.class.getName()));
    }


}
 

@Test
public void readFromMongoDb() throws Exception {
    // populate initial data
    mongo.execInContainer("sh", "-c", "/usr/local/bin/init-inventory.sh");

    Configuration configuration = Configuration
            .create()
            .with("name", "mongodb-inventory-connector")
            .with("connector.class", "io.debezium.connector.mongodb.MongoDbConnector")
            /* begin connector properties */
            .with("mongodb.hosts", "rs0/" + mongo.getContainerIpAddress() + ":"
                    + mongo.getMappedPort(MongoDBContainer.MONGODB_PORT))
            .with("mongodb.name", "fullfillment")
            .with("mongodb.user", "debezium")
            .with("mongodb.password", "dbz")
            .with("mongodb.members.auto.discover", "false")
            .with("collection.whitelist", "inventory.*")
            .with("database.history.hazelcast.list.name", "test")
            .build();

    JetInstance jet = createJetMember();

    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(DebeziumSources.cdc(configuration))
            .withoutTimestamps()
            .map(record -> Values.convertToString(record.valueSchema(), record.value()))
            .writeTo(AssertionSinks.assertCollectedEventually(60,
                    list -> Assert.assertTrue(list.stream().anyMatch(s -> s.contains("Jason")))));

    JobConfig jobConfig = new JobConfig();
    jobConfig.addJarsInZip(Objects.requireNonNull(this.getClass()
                                                      .getClassLoader()
                                                      .getResource("debezium-connector-mongodb.zip")));

    Job job = jet.newJob(pipeline, jobConfig);
    assertJobStatusEventually(job, JobStatus.RUNNING);

    // update record
    mongo.execInContainer("sh", "-c", "/usr/local/bin/insertData.sh");

    try {
        job.join();
        Assert.fail("Job should have completed with an AssertionCompletedException, but completed normally");
    } catch (CompletionException e) {
        String errorMsg = e.getCause().getMessage();
        Assert.assertTrue("Job was expected to complete with AssertionCompletedException, " +
                "but completed with: " + e.getCause(), errorMsg.contains(AssertionCompletedException.class.getName()));
    }


}