Java Code Examples for org.apache.kafka.streams.StreamsBuilder#globalTable()

public AbstractKafkaRepository(String name, Serde<V> valueSerde, Function<V,String> keyFunction, KafkaAdminService kafkaAdminService, KafkaConfiguration kafkaConfiguration) {
    this.repositoryName = name + "-db";
    this.keyFunction = keyFunction;
    this.producer = KafkaUtils.kafkaProducer(kafkaConfiguration.getBootstrapServers(), StringSerializer.class, JsonNodeSerialializer.class);
    kafkaAdminService.createTopic(kafkaAdminService.buildTopicInfo(repositoryName,TopicConfig.CLEANUP_POLICY_COMPACT));

    Properties props = KafkaUtils.createKStreamProperties(repositoryName + "-stream"+ UUID.randomUUID().toString(), kafkaConfiguration.getBootstrapServers());
    StreamsBuilder builder = new StreamsBuilder();

    final GlobalKTable<String, V> globalKTable = builder.globalTable(repositoryName, materialize(valueSerde));

    final KafkaStreams streams = new KafkaStreams(builder.build(), props);
    streams.start();
    producer.flush();
    keyValueStore = streams.store(getStoreName(), QueryableStoreTypes.keyValueStore());

    Runtime.getRuntime().addShutdownHook(new Thread(streams::close));

}
 

private KafkaStreams processStreams(final String bootstrapServers, final String stateDir) throws IOException {

        final StreamsBuilder builder = new StreamsBuilder();

        //Create the streams/tables for the join
        final KStream<String, Order> orders = builder.stream(ORDERS.name(),
                Consumed.with(ORDERS.keySerde(), ORDERS.valueSerde()));
        final GlobalKTable<String, Customer> customers = builder.globalTable(CUSTOMERS.name(),
                Consumed.with(CUSTOMERS.keySerde(), CUSTOMERS.valueSerde()));
//
        // Join a stream and a table then send an email for each
        // GlobalKTable to stream join takes three arguments: Table, mapping of stream (key,value) to table key for join
        // And the join function - takes values from stream and table and returns result
        orders.join(customers,
                        (orderID, order) -> order.getCustomerId(),
                (order, customer) -> new EmailTuple(order,customer))
                //Now for each tuple send an email.
                .peek((key, emailTuple)
                        -> emailer.sendEmail(emailTuple)
                );

        return new KafkaStreams(builder.build(), configStreams(bootstrapServers, stateDir, SERVICE_APP_ID));
    }
 

public Topology getTopologyWithIntermediateTopic() {
    final StreamsBuilder builder = new StreamsBuilder();
    final KStream<Long, Long> inputStream =
            builder.stream(INPUT_TOPIC, Consumed.with(Serdes.Long(), Serdes.Long()));

    builder.stream(NAME_INPUT, Consumed.with(Serdes.Long(), Serdes.String()))
            .mapValues(name -> name.toUpperCase())
            .to(INTERMEDIATE_TOPIC);

    final GlobalKTable<Long, String> joinTable = builder.globalTable(INTERMEDIATE_TOPIC);

    inputStream
            .join(joinTable,
                    (id, valueId) -> valueId,
                    (id, name) -> name)
            .to(OUTPUT_TOPIC, Produced.with(Serdes.Long(), Serdes.String()));

    return builder.build();
}
 

public Topology getTopology() {
    final StreamsBuilder builder = new StreamsBuilder();
    final KStream<Long, Long> inputStream =
            builder.stream(INPUT_TOPIC, Consumed.with(Serdes.Long(), Serdes.Long()));

    final GlobalKTable<Long, String> joinTable = builder.globalTable(NAME_INPUT);

    inputStream
            .join(joinTable,
                    (id, valueId) -> valueId,
                    (id, name) -> name)
            .to(OUTPUT_TOPIC, Produced.with(Serdes.Long(), Serdes.String()));

    return builder.build();
}
 

@Produces
public Topology buildTopology() {
    StreamsBuilder builder = new StreamsBuilder();

    JsonbSerde<WeatherStation> weatherStationSerde = new JsonbSerde<>(WeatherStation.class);
    JsonbSerde<Aggregation> aggregationSerde = new JsonbSerde<>(Aggregation.class);

    KeyValueBytesStoreSupplier storeSupplier = Stores.persistentKeyValueStore(WEATHER_STATIONS_STORE);

    GlobalKTable<Integer, WeatherStation> stations = builder.globalTable(
            WEATHER_STATIONS_TOPIC,
            Consumed.with(Serdes.Integer(), weatherStationSerde));

    builder.stream(
            TEMPERATURE_VALUES_TOPIC,
            Consumed.with(Serdes.Integer(), Serdes.String()))
            .join(
                    stations,
                    (stationId, timestampAndValue) -> stationId,
                    (timestampAndValue, station) -> {
                        String[] parts = timestampAndValue.split(";");
                        return new TemperatureMeasurement(station.id, station.name, Instant.parse(parts[0]),
                                Double.valueOf(parts[1]));
                    })
            .groupByKey()
            .aggregate(
                    Aggregation::new,
                    (stationId, value, aggregation) -> aggregation.updateFrom(value),
                    Materialized.<Integer, Aggregation> as(storeSupplier)
                            .withKeySerde(Serdes.Integer())
                            .withValueSerde(aggregationSerde))
            .toStream()
            .to(
                    TEMPERATURES_AGGREGATED_TOPIC,
                    Produced.with(Serdes.Integer(), aggregationSerde));

    return builder.build();
}
 

private <K, V> GlobalKTable<K, V> materializedAsGlobalKTable(
		StreamsBuilder streamsBuilder, String destination, String storeName,
		Serde<K> k, Serde<V> v, Topology.AutoOffsetReset autoOffsetReset, KafkaStreamsConsumerProperties kafkaStreamsConsumerProperties) {
	final Consumed<K, V> consumed = getConsumed(kafkaStreamsConsumerProperties, k, v, autoOffsetReset);
	return streamsBuilder.globalTable(
			this.bindingServiceProperties.getBindingDestination(destination),
			consumed,
			getMaterialized(storeName, k, v));
}
 

private GlobalKTable<?, ?> getGlobalKTable(KafkaStreamsConsumerProperties kafkaStreamsConsumerProperties,
		StreamsBuilder streamsBuilder,
		Serde<?> keySerde, Serde<?> valueSerde, String materializedAs,
		String bindingDestination, Topology.AutoOffsetReset autoOffsetReset) {
	final Consumed<?, ?> consumed = getConsumed(kafkaStreamsConsumerProperties, keySerde, valueSerde, autoOffsetReset);
	return materializedAs != null
			? materializedAsGlobalKTable(streamsBuilder, bindingDestination,
			materializedAs, keySerde, valueSerde, autoOffsetReset, kafkaStreamsConsumerProperties)
			: streamsBuilder.globalTable(bindingDestination,
			consumed);
}
 

public static void main(String[] args) {
    Properties props = new Properties();
    props.put(StreamsConfig.APPLICATION_ID_CONFIG, AppConfigs.applicationID);
    props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, AppConfigs.bootstrapServers);
    props.put(StreamsConfig.STATE_DIR_CONFIG, AppConfigs.stateStoreLocation);

    StreamsBuilder streamsBuilder = new StreamsBuilder();

    //Global table for mappings
    GlobalKTable<String, TableMap> topicTableMapGlobalKTable = streamsBuilder.globalTable(
        AppConfigs.topicTableMap,
        Consumed.with(AppSerdes.String(),
            AppSerdes.TableMap()));

    //Stream of Records
    KStream<String, GenericRecord> recordKStream = streamsBuilder.stream(
        AppConfigs.topicPattern,
        Consumed.with(AppSerdes.String(),
            AppSerdes.GenericRecord())
    ).transform(() -> new RecordTransformer());

    //Join to get Target Table Name
    KStream<String, GenericRecord> joinedKStream = recordKStream.join(
        topicTableMapGlobalKTable,
        (keyGenericRecord, valueGenericRecord) -> keyGenericRecord,
        (valueGenericRecord, valueTableMap) -> {
            valueGenericRecord.setAdditionalProperty(
                AppConfigs.targetTableField,
                valueTableMap.getTargetTable());
            return valueGenericRecord;
        }
    );

    //Change key to target table name and cleanup record
    KStream<String, GenericRecord> sinkRecord = joinedKStream.selectKey(
        (k, v) -> {
            String newKey = v.getAdditionalProperties()
                .get(AppConfigs.targetTableField);
            v.getAdditionalProperties().remove(AppConfigs.targetTableField);
            return newKey;
        }).peek((k, v) -> logger.info("Ready to Sink key= " + k + " value= " + v));

    //Sink to Target Database
    sinkRecord.process(() -> new SinkProcessor());

    //Start the stream and add a shutdown hook
    KafkaStreams streams = new KafkaStreams(streamsBuilder.build(), props);
    streams.start();
    Runtime.getRuntime().addShutdownHook(new Thread(streams::close));
}
 

public static void main(String[] args) {
    Properties properties = new Properties();
    properties.put(StreamsConfig.APPLICATION_ID_CONFIG,
        AppConfigs.applicationID);
    properties.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG,
        AppConfigs.bootstrapServers);
    properties.put(StreamsConfig.STATE_DIR_CONFIG,
        AppConfigs.stateStoreName);

    StreamsBuilder streamsBuilder = new StreamsBuilder();

    //Global table for all inventories
    GlobalKTable<String, AdInventories> adInventoriesGlobalKTable =
        streamsBuilder.globalTable(
            AppConfigs.inventoryTopic,
            Consumed.with(AppSerdes.String(),
                AppSerdes.AdInventories())
        );

    //Stream of Clicks
    KStream<String, AdClick> clicksKStream =
        streamsBuilder.stream(AppConfigs.clicksTopic,
            Consumed.with(AppSerdes.String(),
                AppSerdes.AdClick())
        );

    //Stream of Clicked Inventories
    KStream<String, AdInventories> clickedInventoryKStream =
        clicksKStream.join(
            adInventoriesGlobalKTable,
            (clickKey, clickValue) -> clickKey,
            (adClick, adInventory) -> adInventory
        );

    //Group clicked inventories on NewsType and count
    //You will get clicks by news Types
    KTable<String, Long> clicksByNewsTypeKTable =
        clickedInventoryKStream.groupBy(
            (inventoryID, inventoryValue) -> inventoryValue.getNewsType(),
            Grouped.with(AppSerdes.String(), AppSerdes.AdInventories())
        ).count();

    //The clicksByNewsType is exhaustive and distributed
    //We need to compute the top 3 only
    //In order to do that, we must following
    //1. Bring all clicksByNewsType to a single partition
    //2. Sort them by clicks and take only top 3
    //There are two steps to achieve this.
    //1. Group them on a common key to bring it to a single partition
    //2. Simulate Sort()+Top(3) using a custom aggregation
    KTable<String, Top3NewsTypes> top3NewsTypesKTable =
        clicksByNewsTypeKTable.groupBy(
            (k_newsType, v_clickCount) -> {
                ClicksByNewsType value = new ClicksByNewsType();
                value.setNewsType(k_newsType);
                value.setClicks(v_clickCount);
                return KeyValue.pair(AppConfigs.top3AggregateKey, value);
            },
            Grouped.with(AppSerdes.String(), AppSerdes.ClicksByNewsType())
        ).aggregate(Top3NewsTypes::new,
            (k, newClickByNewsType, aggTop3NewType) -> {
                aggTop3NewType.add(newClickByNewsType);
                return aggTop3NewType;
            },
            (k, oldClickByNewsType, aggTop3NewType) -> {
                aggTop3NewType.remove(oldClickByNewsType);
                return aggTop3NewType;
            },
            Materialized.<String, Top3NewsTypes, KeyValueStore<Bytes, byte[]>>
                as("top3-clicks")
                .withKeySerde(AppSerdes.String())
                .withValueSerde(AppSerdes.Top3NewsTypes()));

    top3NewsTypesKTable.toStream().foreach((k, v) -> {
        try {
            logger.info("k=" + k + " v= " + v.getTop3Sorted());
        } catch (JsonProcessingException e) {
            e.printStackTrace();
        }
    });

    KafkaStreams streams = new KafkaStreams(streamsBuilder.build(), properties);
    streams.start();
    Runtime.getRuntime().addShutdownHook(new Thread(streams::close));
}
 

public static void main(String[] args) throws Exception {


        StreamsConfig streamsConfig = new StreamsConfig(getProperties());

        Serde<String> stringSerde = Serdes.String();
        Serde<StockTransaction> transactionSerde = StreamsSerdes.StockTransactionSerde();
        Serde<TransactionSummary> transactionSummarySerde = StreamsSerdes.TransactionSummarySerde();


        StreamsBuilder builder = new StreamsBuilder();
        long twentySeconds = 1000 * 20;

        KeyValueMapper<Windowed<TransactionSummary>, Long, KeyValue<String, TransactionSummary>> transactionMapper = (window, count) -> {
            TransactionSummary transactionSummary = window.key();
            String newKey = transactionSummary.getIndustry();
            transactionSummary.setSummaryCount(count);
            return KeyValue.pair(newKey, transactionSummary);
        };

        KStream<String, TransactionSummary> countStream =
                builder.stream( STOCK_TRANSACTIONS_TOPIC, Consumed.with(stringSerde, transactionSerde).withOffsetResetPolicy(LATEST))
                        .groupBy((noKey, transaction) -> TransactionSummary.from(transaction), Serialized.with(transactionSummarySerde, transactionSerde))
                        .windowedBy(SessionWindows.with(twentySeconds)).count()
                        .toStream().map(transactionMapper);

        GlobalKTable<String, String> publicCompanies = builder.globalTable(COMPANIES.topicName());
        GlobalKTable<String, String> clients = builder.globalTable(CLIENTS.topicName());


        countStream.leftJoin(publicCompanies, (key, txn) -> txn.getStockTicker(),TransactionSummary::withCompanyName)
                .leftJoin(clients, (key, txn) -> txn.getCustomerId(), TransactionSummary::withCustomerName)
                .print(Printed.<String, TransactionSummary>toSysOut().withLabel("Resolved Transaction Summaries"));


        
        KafkaStreams kafkaStreams = new KafkaStreams(builder.build(), streamsConfig);
        kafkaStreams.cleanUp();


        kafkaStreams.setUncaughtExceptionHandler((t, e) -> {
            LOG.error("had exception ", e);
        });

        CustomDateGenerator dateGenerator = CustomDateGenerator.withTimestampsIncreasingBy(Duration.ofMillis(750));

        DataGenerator.setTimestampGenerator(dateGenerator::get);

        MockDataProducer.produceStockTransactions(2, 5, 3, true);

        LOG.info("Starting GlobalKTable Example");
        kafkaStreams.cleanUp();
        kafkaStreams.start();
        Thread.sleep(65000);
        LOG.info("Shutting down the GlobalKTable Example Application now");
        kafkaStreams.close();
        MockDataProducer.shutdown();
    }