org.apache.kafka.streams.kstream.KStream#process

Source File: KafkaStreamsStateStoreIntegrationTests.java From spring-cloud-stream-binder-kafka with Apache License 2.0

6 votes

@StreamListener("input")
@KafkaStreamsStateStore(name = "mystate", type = KafkaStreamsStateStoreProperties.StoreType.WINDOW, lengthMs = 300000, retentionMs = 300000)
@SuppressWarnings({ "deprecation", "unchecked" })
public void process(KStream<Object, Product> input) {

	input.process(() -> new Processor<Object, Product>() {

		@Override
		public void init(ProcessorContext processorContext) {
			state = (WindowStore) processorContext.getStateStore("mystate");
		}

		@Override
		public void process(Object s, Product product) {
			processed = true;
		}

		@Override
		public void close() {
			if (state != null) {
				state.close();
			}
		}
	}, "mystate");
}

Source File: KafkaStreamsStateStoreIntegrationTests.java From spring-cloud-stream-binder-kafka with Apache License 2.0

6 votes

@StreamListener("input3")
@SuppressWarnings({"unchecked" })
public void process(KStream<Object, Product> input) {

	input.process(() -> new Processor<Object, Product>() {

		@Override
		public void init(ProcessorContext processorContext) {
			state = (WindowStore) processorContext.getStateStore("mystate");
		}

		@Override
		public void process(Object s, Product product) {
			processed = true;
		}

		@Override
		public void close() {
			if (state != null) {
				state.close();
			}
		}
	}, "mystate");
}

Source File: ProcessStreamService.java From SkaETL with Apache License 2.0

5 votes

public void createStreamEs(String inputTopic, ParameterOutput parameterOutput) {

        StreamsBuilder builder = new StreamsBuilder();

        KStream<String, JsonNode> streamToES = builder.stream(inputTopic, Consumed.with(Serdes.String(), GenericSerdes.jsonNodeSerde()));
        streamToES.process(() -> applicationContext.getBean(JsonNodeToElasticSearchProcessor.class, parameterOutput.getElasticsearchRetentionLevel(), parameterOutput.getIndexShape()));

        KafkaStreams streams = new KafkaStreams(builder.build(), KafkaUtils.createKStreamProperties(getProcessConsumer().getIdProcess() + ProcessConstants.ES_PROCESS, getBootstrapServer()));
        Runtime.getRuntime().addShutdownHook(new Thread(streams::close));
        streams.start();
        addStreams(streams);
    }

Source File: ErrorImporter.java From SkaETL with Apache License 2.0

5 votes

public void activate() {
    log.info("Activating error importer");
    StreamsBuilder builder = new StreamsBuilder();
    final Serde<ErrorData> errorDataSerde = Serdes.serdeFrom(new GenericSerializer<>(), new GenericDeserializer<>(ErrorData.class));

    KStream<String, ErrorData> streamToES = builder.stream(kafkaConfiguration.getErrorTopic(), Consumed.with(Serdes.String(), errorDataSerde));

    streamToES.process(() -> elasticsearchProcessor);

    errorStream = new KafkaStreams(builder.build(), KafkaUtils.createKStreamProperties(INPUT_PROCESS_ERROR, kafkaConfiguration.getBootstrapServers()));
    Runtime.getRuntime().addShutdownHook(new Thread(errorStream::close));

    errorStream.start();
}

Source File: RetryImporter.java From SkaETL with Apache License 2.0

5 votes

public void activate() {
    log.info("Activating retry importer");
    StreamsBuilder builder = new StreamsBuilder();
    final Serde<ValidateData> validateDataSerdes = Serdes.serdeFrom(new ValidateDataSerializer(), new ValidateDataDeserializer());

    KStream<String, ValidateData> streamToES = builder.stream(kafkaConfiguration.getRetryTopic(), Consumed.with(Serdes.String(), validateDataSerdes));
    streamToES.process(() -> elasticSearchProcessor);

    retryStream = new KafkaStreams(builder.build(), KafkaUtils.createKStreamProperties(INPUT_PROCESS_RETRY, kafkaConfiguration.getBootstrapServers()));
    Runtime.getRuntime().addShutdownHook(new Thread(retryStream::close));
    retryStream.start();
}

Source File: KafkaStreamsStateStoreIntegrationTests.java From spring-cloud-stream-binder-kafka with Apache License 2.0

5 votes

@StreamListener
@KafkaStreamsStateStore(name = "mystate", type = KafkaStreamsStateStoreProperties.StoreType.WINDOW, lengthMs = 300000, retentionMs = 300000)
@SuppressWarnings({ "deprecation", "unchecked" })
public void process(@Input("input1")KStream<Object, Product> input, @Input("input2")KStream<Object, Product> input2) {

	input.process(() -> new Processor<Object, Product>() {

		@Override
		public void init(ProcessorContext processorContext) {
			state = (WindowStore) processorContext.getStateStore("mystate");
		}

		@Override
		public void process(Object s, Product product) {
			processed = true;
		}

		@Override
		public void close() {
			if (state != null) {
				state.close();
			}
		}
	}, "mystate");

	//simple use of input2, we are not using input2 for anything other than triggering some test behavior.
	input2.foreach((key, value) -> { });
}

Source File: CommandProcessor.java From cqrs-manager-for-distributed-reactive-services with Apache License 2.0

5 votes

public void start() {
    KStreamBuilder builder = new KStreamBuilder();

    Serde<UUID> keySerde = new FressianSerde();
    Serde<Map> valSerde = new FressianSerde();

    KStream<UUID, Map> commands = builder.stream(keySerde, valSerde, commandsTopic);
    KStream<UUID, Map> customerEvents = commands
            .filter((id, command) -> command.get(new Keyword("action")).equals(new Keyword("create-customer")))
            .map((id, command) -> {
                logger.debug("Command received");
                Map userEvent = new HashMap(command);
                userEvent.put(new Keyword("action"), new Keyword("customer-created"));
                userEvent.put(new Keyword("parent"), id);
                Map userValue = (Map) userEvent.get(new Keyword("data"));
                userValue.put(new Keyword("id"), UUID.randomUUID());
                return new KeyValue<>(UUID.randomUUID(), userEvent);
    }).through(keySerde, valSerde, eventsTopic);

    KStream<UUID, Map> customers = customerEvents
            .map((id, event) -> {
                Map customer = (Map) event.get(new Keyword("data"));
                UUID customerId = (UUID) customer.get(new Keyword("id"));
                return new KeyValue<UUID, Map>(customerId, customer);
            });

    customers.through(keySerde, valSerde, customersTopic);

    StateStoreSupplier store = Stores.create("Customers")
            .withKeys(keySerde)
            .withValues(valSerde)
            .persistent()
            .build();
    builder.addStateStore(store);

    customers.process(customerStore, "Customers");

    this.kafkaStreams = new KafkaStreams(builder, kafkaStreamsConfig);
    this.kafkaStreams.start();
}

Source File: StreamsTopologyProvider.java From apicurio-registry with Apache License 2.0

4 votes

@Override
public Topology get() {
    StreamsBuilder builder = new StreamsBuilder();

    // Simple defaults
    ImmutableMap<String, String> configuration = ImmutableMap.of(
        TopicConfig.CLEANUP_POLICY_CONFIG, TopicConfig.CLEANUP_POLICY_COMPACT,
        TopicConfig.MIN_COMPACTION_LAG_MS_CONFIG, "0",
        TopicConfig.SEGMENT_BYTES_CONFIG, String.valueOf(64 * 1024 * 1024)
    );

    // Input topic -- storage topic
    // This is where we handle "http" requests
    // Key is artifactId -- which is also used for KeyValue store key
    KStream<String, Str.StorageValue> storageRequest = builder.stream(
        properties.getStorageTopic(),
        Consumed.with(Serdes.String(), ProtoSerde.parsedWith(Str.StorageValue.parser()))
    );

    // Data structure holds all artifact information
    // Global rules are Data as well, with constant artifactId (GLOBAL_RULES variable)
    String storageStoreName = properties.getStorageStoreName();
    StoreBuilder<KeyValueStore<String /* artifactId */, Str.Data>> storageStoreBuilder =
        Stores
            .keyValueStoreBuilder(
                Stores.inMemoryKeyValueStore(storageStoreName),
                Serdes.String(), ProtoSerde.parsedWith(Str.Data.parser())
            )
            .withCachingEnabled()
            .withLoggingEnabled(configuration);

    builder.addStateStore(storageStoreBuilder);

    // We transform <artifactId, Data> into simple mapping <globalId, <artifactId, version>>
    KStream<Long, Str.TupleValue> globalRequest =
        storageRequest.transform(
            () -> new StorageTransformer(properties, dataDispatcher, factory),
            storageStoreName
        ).through(
            properties.getGlobalIdTopic(),
            Produced.with(Serdes.Long(), ProtoSerde.parsedWith(Str.TupleValue.parser()))
        );

    String globalIdStoreName = properties.getGlobalIdStoreName();
    StoreBuilder<KeyValueStore<Long /* globalId */, Str.TupleValue>> globalIdStoreBuilder =
        Stores
            .keyValueStoreBuilder(
                Stores.inMemoryKeyValueStore(globalIdStoreName),
                Serdes.Long(), ProtoSerde.parsedWith(Str.TupleValue.parser())
            )
            .withCachingEnabled()
            .withLoggingEnabled(configuration);

    builder.addStateStore(globalIdStoreBuilder);

    // Just handle globalId mapping -- put or delete
    globalRequest.process(() -> new GlobalIdProcessor(globalIdStoreName), globalIdStoreName);

    return builder.build(properties.getProperties());
}