Java Code Examples for org.apache.kafka.streams.Topology#addProcessor()
The following examples show how to use
org.apache.kafka.streams.Topology#addProcessor() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: KStreamsTopologyDescriptionParserTest.java From netbeans-mmd-plugin with Apache License 2.0 | 7 votes |
|
@Test
public void testFromTopology() {
final Topology topology = new Topology();
topology.addSource("SomeSource", "topic1", "topic2");
topology.addProcessor("Processor1", () -> new FakeProcessor(), "SomeSource");
topology.addProcessor("Processor2", () -> new FakeProcessor(), "Processor1");
topology.addProcessor("Processor3", () -> new FakeProcessor(), "Processor2");
topology.addSink("TheSink", "FinalTopic", "Processor3");
final String src = topology.describe().toString();
System.out.println(src);
final KStreamsTopologyDescriptionParser graph = new KStreamsTopologyDescriptionParser(src);
System.out.println(graph);
assertEquals(5, graph.size());
}
Example 2
| Source File: StockPerformanceApplication.java From kafka-streams-in-action with Apache License 2.0 | 5 votes |
|
public static void main(String[] args) throws Exception {
StreamsConfig streamsConfig = new StreamsConfig(getProperties());
Deserializer<String> stringDeserializer = Serdes.String().deserializer();
Serializer<String> stringSerializer = Serdes.String().serializer();
Serde<StockPerformance> stockPerformanceSerde = StreamsSerdes.StockPerformanceSerde();
Serializer<StockPerformance> stockPerformanceSerializer = stockPerformanceSerde.serializer();
Serde<StockTransaction> stockTransactionSerde = StreamsSerdes.StockTransactionSerde();
Deserializer<StockTransaction> stockTransactionDeserializer = stockTransactionSerde.deserializer();
Topology topology = new Topology();
String stocksStateStore = "stock-performance-store";
double differentialThreshold = 0.02;
KeyValueBytesStoreSupplier storeSupplier = Stores.inMemoryKeyValueStore(stocksStateStore);
StoreBuilder<KeyValueStore<String, StockPerformance>> storeBuilder = Stores.keyValueStoreBuilder(storeSupplier, Serdes.String(), stockPerformanceSerde);
topology.addSource("stocks-source", stringDeserializer, stockTransactionDeserializer,"stock-transactions")
.addProcessor("stocks-processor", () -> new StockPerformanceProcessor(stocksStateStore, differentialThreshold), "stocks-source")
.addStateStore(storeBuilder,"stocks-processor")
.addSink("stocks-sink", "stock-performance", stringSerializer, stockPerformanceSerializer, "stocks-processor");
topology.addProcessor("stocks-printer", new KStreamPrinter("StockPerformance"), "stocks-processor");
KafkaStreams kafkaStreams = new KafkaStreams(topology, streamsConfig);
MockDataProducer.produceStockTransactionsWithKeyFunction(50,50, 25, StockTransaction::getSymbol);
System.out.println("Stock Analysis App Started");
kafkaStreams.cleanUp();
kafkaStreams.start();
Thread.sleep(70000);
System.out.println("Shutting down the Stock Analysis App now");
kafkaStreams.close();
MockDataProducer.shutdown();
}
Example 3
| Source File: PopsHopsApplication.java From kafka-streams-in-action with Apache License 2.0 | 4 votes |
|
public static void main(String[] args) throws Exception {
StreamsConfig streamsConfig = new StreamsConfig(getProperties());
Deserializer<BeerPurchase> beerPurchaseDeserializer = new JsonDeserializer<>(BeerPurchase.class);
Serde<String> stringSerde = Serdes.String();
Deserializer<String> stringDeserializer = stringSerde.deserializer();
Serializer<String> stringSerializer = stringSerde.serializer();
Serializer<BeerPurchase> beerPurchaseSerializer = new JsonSerializer<>();
Topology toplogy = new Topology();
String domesticSalesSink = "domestic-beer-sales";
String internationalSalesSink = "international-beer-sales";
String purchaseSourceNodeName = "beer-purchase-source";
String purchaseProcessor = "purchase-processor";
BeerPurchaseProcessor beerProcessor = new BeerPurchaseProcessor(domesticSalesSink, internationalSalesSink);
toplogy.addSource(LATEST,
purchaseSourceNodeName,
new UsePreviousTimeOnInvalidTimestamp(),
stringDeserializer,
beerPurchaseDeserializer,
Topics.POPS_HOPS_PURCHASES.topicName())
.addProcessor(purchaseProcessor,
() -> beerProcessor,
purchaseSourceNodeName);
//Uncomment these two lines and comment out the printer lines for writing to topics
// .addSink(internationalSalesSink,"international-sales", stringSerializer, beerPurchaseSerializer, purchaseProcessor)
// .addSink(domesticSalesSink,"domestic-sales", stringSerializer, beerPurchaseSerializer, purchaseProcessor);
//You'll have to comment these lines out if you want to write to topics as they have the same node names
toplogy.addProcessor(domesticSalesSink,
new KStreamPrinter("domestic-sales"),
purchaseProcessor );
toplogy.addProcessor(internationalSalesSink,
new KStreamPrinter("international-sales"),
purchaseProcessor );
KafkaStreams kafkaStreams = new KafkaStreams(toplogy, streamsConfig);
MockDataProducer.produceBeerPurchases(5);
System.out.println("Starting Pops-Hops Application now");
kafkaStreams.cleanUp();
kafkaStreams.start();
Thread.sleep(70000);
System.out.println("Shutting down Pops-Hops Application now");
kafkaStreams.close();
MockDataProducer.shutdown();
}
Example 4
| Source File: CoGroupingApplication.java From kafka-streams-in-action with Apache License 2.0 | 4 votes |
|
public static void main(String[] args) throws Exception {
StreamsConfig streamsConfig = new StreamsConfig(getProperties());
Deserializer<String> stringDeserializer = Serdes.String().deserializer();
Serializer<String> stringSerializer = Serdes.String().serializer();
Serde<Tuple<List<ClickEvent>, List<StockTransaction>>> eventPerformanceTuple = StreamsSerdes.EventTransactionTupleSerde();
Serializer<Tuple<List<ClickEvent>, List<StockTransaction>>> tupleSerializer = eventPerformanceTuple.serializer();
Serde<StockTransaction> stockTransactionSerde = StreamsSerdes.StockTransactionSerde();
Deserializer<StockTransaction> stockTransactionDeserializer = stockTransactionSerde.deserializer();
Serde<ClickEvent> clickEventSerde = StreamsSerdes.ClickEventSerde();
Deserializer<ClickEvent> clickEventDeserializer = clickEventSerde.deserializer();
Topology topology = new Topology();
Map<String, String> changeLogConfigs = new HashMap<>();
changeLogConfigs.put("retention.ms", "120000");
changeLogConfigs.put("cleanup.policy", "compact,delete");
KeyValueBytesStoreSupplier storeSupplier = Stores.persistentKeyValueStore(TUPLE_STORE_NAME);
StoreBuilder<KeyValueStore<String, Tuple<List<ClickEvent>, List<StockTransaction>>>> storeBuilder =
Stores.keyValueStoreBuilder(storeSupplier,
Serdes.String(),
eventPerformanceTuple).withLoggingEnabled(changeLogConfigs);
topology.addSource("Txn-Source", stringDeserializer, stockTransactionDeserializer, "stock-transactions")
.addSource("Events-Source", stringDeserializer, clickEventDeserializer, "events")
.addProcessor("Txn-Processor", StockTransactionProcessor::new, "Txn-Source")
.addProcessor("Events-Processor", ClickEventProcessor::new, "Events-Source")
.addProcessor("CoGrouping-Processor", CogroupingProcessor::new, "Txn-Processor", "Events-Processor")
.addStateStore(storeBuilder, "CoGrouping-Processor")
.addSink("Tuple-Sink", "cogrouped-results", stringSerializer, tupleSerializer, "CoGrouping-Processor");
topology.addProcessor("Print", new KStreamPrinter("Co-Grouping"), "CoGrouping-Processor");
MockDataProducer.produceStockTransactionsAndDayTradingClickEvents(50, 100, 100, StockTransaction::getSymbol);
KafkaStreams kafkaStreams = new KafkaStreams(topology, streamsConfig);
System.out.println("Co-Grouping App Started");
kafkaStreams.cleanUp();
kafkaStreams.start();
Thread.sleep(70000);
System.out.println("Shutting down the Co-Grouping App now");
kafkaStreams.close();
MockDataProducer.shutdown();
}
Example 5
| Source File: CoGroupingListeningExampleApplication.java From kafka-streams-in-action with Apache License 2.0 | 4 votes |
|
public static void main(String[] args) throws Exception {
StreamsConfig streamsConfig = new StreamsConfig(getProperties());
Deserializer<String> stringDeserializer = Serdes.String().deserializer();
Serializer<String> stringSerializer = Serdes.String().serializer();
Serde<Tuple<List<ClickEvent>, List<StockTransaction>>> eventPerformanceTuple = StreamsSerdes.EventTransactionTupleSerde();
Serializer<Tuple<List<ClickEvent>, List<StockTransaction>>> tupleSerializer = eventPerformanceTuple.serializer();
Serde<StockTransaction> stockTransactionSerde = StreamsSerdes.StockTransactionSerde();
Deserializer<StockTransaction> stockTransactionDeserializer = stockTransactionSerde.deserializer();
Serde<ClickEvent> clickEventSerde = StreamsSerdes.ClickEventSerde();
Deserializer<ClickEvent> clickEventDeserializer = clickEventSerde.deserializer();
Topology topology = new Topology();
Map<String, String> changeLogConfigs = new HashMap<>();
changeLogConfigs.put("retention.ms","120000" );
changeLogConfigs.put("cleanup.policy", "compact,delete");
KeyValueBytesStoreSupplier storeSupplier = Stores.persistentKeyValueStore(TUPLE_STORE_NAME);
StoreBuilder<KeyValueStore<String, Tuple<List<ClickEvent>, List<StockTransaction>>>> builder = Stores.keyValueStoreBuilder(storeSupplier, Serdes.String(), eventPerformanceTuple);
topology.addSource("Txn-Source", stringDeserializer, stockTransactionDeserializer, "stock-transactions")
.addSource( "Events-Source", stringDeserializer, clickEventDeserializer, "events")
.addProcessor("Txn-Processor", StockTransactionProcessor::new, "Txn-Source")
.addProcessor("Events-Processor", ClickEventProcessor::new, "Events-Source")
.addProcessor("CoGrouping-Processor", CogroupingProcessor::new, "Txn-Processor", "Events-Processor")
.addStateStore(builder.withLoggingEnabled(changeLogConfigs), "CoGrouping-Processor")
.addSink("Tuple-Sink", "cogrouped-results", stringSerializer, tupleSerializer, "CoGrouping-Processor");
topology.addProcessor("Print", new KStreamPrinter("Co-Grouping"), "CoGrouping-Processor");
MockDataProducer.produceStockTransactionsAndDayTradingClickEvents(50, 100, 100, StockTransaction::getSymbol);
KafkaStreams kafkaStreams = new KafkaStreams(topology, streamsConfig);
kafkaStreams.setGlobalStateRestoreListener(new LoggingStateRestoreListener());
kafkaStreams.setUncaughtExceptionHandler((thread, exception) ->
LOG.error("Thread [{}] encountered [{}]", thread.getName(), exception.getMessage())
);
kafkaStreams.setStateListener((newState, oldState) -> {
if (oldState == KafkaStreams.State.REBALANCING && newState== KafkaStreams.State.RUNNING) {
LOG.info("Topology Layout {}", topology.describe());
LOG.info("Thread metadata {}", kafkaStreams.localThreadsMetadata());
}
});
LOG.info("Co-Grouping App Started");
kafkaStreams.cleanUp();
kafkaStreams.start();
Thread.sleep(70000);
LOG.info("Shutting down the Co-Grouping metrics App now");
kafkaStreams.close();
MockDataProducer.shutdown();
}
Example 6
| Source File: StockFeedApp.java From Kafka-Streams-Real-time-Stream-Processing with The Unlicense | 4 votes |
|
public static void main(String[] args) {
Properties props = new Properties();
props.put(StreamsConfig.APPLICATION_ID_CONFIG, AppConfigs.applicationID);
props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, AppConfigs.bootstrapServers);
props.put(StreamsConfig.STATE_DIR_CONFIG, AppConfigs.stateStoreLocation);
Topology topology = new Topology();
topology.addSource(
AppConfigs.sourceName,
AppSerdes.String().deserializer(),
AppSerdes.StockTicker().deserializer(),
AppConfigs.sourceTopicName);
topology.addProcessor(
StockProcessor.PROCESSOR_NAME,
() -> new StockProcessor(),
AppConfigs.sourceName);
StoreBuilder storeBuilder = Stores.keyValueStoreBuilder(
Stores.persistentKeyValueStore(StockProcessor.STATE_STORE_NAME),
AppSerdes.String(),
AppSerdes.TickerStack()
);
topology.addStateStore(
storeBuilder,
StockProcessor.PROCESSOR_NAME);
topology.addSink(AppConfigs.sinkName,
AppConfigs.targetTopicName,
AppSerdes.String().serializer(),
AppSerdes.StockTicker().serializer(),
StockProcessor.PROCESSOR_NAME);
KafkaStreams streams = new KafkaStreams(topology, props);
streams.start();
Runtime.getRuntime().addShutdownHook(new Thread(streams::close));
}
