org.apache.kafka.common.utils.Bytes Java Examples
The following examples show how to use
org.apache.kafka.common.utils.Bytes.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ScsApplication.java From spring_io_2019 with Apache License 2.0 | 7 votes |
@StreamListener @SendTo(Bindings.RATED_MOVIES) KStream<Long, RatedMovie> rateMoviesFor(@Input(Bindings.AVG_TABLE) KTable<Long, Double> ratings, @Input(Bindings.MOVIES) KTable<Long, Movie> movies) { ValueJoiner<Movie, Double, RatedMovie> joiner = (movie, rating) -> new RatedMovie(movie.getMovieId(), movie.getReleaseYear(), movie.getTitle(), rating); movies .join(ratings, joiner, Materialized .<Long, RatedMovie, KeyValueStore<Bytes, byte[]>>as(Bindings.RATED_MOVIES_STORE) .withKeySerde(Serdes.Long()) .withValueSerde(new JsonSerde<>(RatedMovie.class))); return movies.join(ratings, joiner).toStream(); }
Example #2
Source File: TopicStreamWriterFormatTest.java From ksql-fork-with-deep-learning-function with Apache License 2.0 | 6 votes |
@Test public void shouldMatchJsonFormatter() throws Exception { SchemaRegistryClient schemaRegistryClient = mock(SchemaRegistryClient.class); replay(schemaRegistryClient); /** * Test data */ String json = "{ \"name\": \"myrecord\"," + " \"type\": \"record\"" + "}"; ConsumerRecord<String, Bytes> record = new ConsumerRecord<String, Bytes>("topic", 1, 1, "key", new Bytes(json.getBytes())); assertTrue(TopicStreamWriter.Format.JSON.isFormat("topic", record, schemaRegistryClient)); }
Example #3
Source File: TumblingWindowExpressionTest.java From ksql-fork-with-deep-learning-function with Apache License 2.0 | 6 votes |
@Test public void shouldCreateTumblingWindowAggregate() { final KGroupedStream stream = EasyMock.createNiceMock(KGroupedStream.class); final TimeWindowedKStream windowedKStream = EasyMock.createNiceMock(TimeWindowedKStream.class); final UdafAggregator aggregator = EasyMock.createNiceMock(UdafAggregator.class); final TumblingWindowExpression windowExpression = new TumblingWindowExpression(10, TimeUnit.SECONDS); final Initializer initializer = () -> 0; final Materialized<String, GenericRow, WindowStore<Bytes, byte[]>> store = Materialized.as("store"); EasyMock.expect(stream.windowedBy(TimeWindows.of(10000L))).andReturn(windowedKStream); EasyMock.expect(windowedKStream.aggregate(same(initializer), same(aggregator), same(store))).andReturn(null); EasyMock.replay(stream, windowedKStream); windowExpression.applyAggregate(stream, initializer, aggregator, store); EasyMock.verify(stream, windowedKStream); }
Example #4
Source File: KafkaMetricReceiver.java From super-cloudops with Apache License 2.0 | 6 votes |
/** * Receiving consumer messages on multiple topics * * @param records * @param ack */ @KafkaListener(topicPattern = TOPIC_KAFKA_RECEIVE_PATTERN, containerFactory = BEAN_KAFKA_BATCH_FACTORY) public void onMetricReceive(List<ConsumerRecord<byte[], Bytes>> records, Acknowledgment ack) { try { if (log.isDebugEnabled()) { log.debug("Receive metric records - {}", records); } if (log.isInfoEnabled()) { log.info("Receive metric records size - {}", records.size()); } doProcess(records, new MultiAcknowledgmentState(ack)); } catch (Exception e) { log.error(String.format("Failed to receive process for ", records.size()), e); } }
Example #5
Source File: KafkaMetricReceiver.java From super-cloudops with Apache License 2.0 | 6 votes |
/** * UMC agent metric processing. * * @param records * @param state */ private void doProcess(List<ConsumerRecord<byte[], Bytes>> records, MultiAcknowledgmentState state) { for (ConsumerRecord<byte[], Bytes> record : records) { try { MetricAggregate aggregate = MetricAggregate.parseFrom(record.value().get()); if (log.isDebugEnabled()) { log.debug("Put metric aggregate for - {}", aggregate); } // Storage metrics. putMetrics(aggregate); // Metrics alarm. alarm(aggregate); } catch (InvalidProtocolBufferException e) { log.error("Failed to parse metric message.", e); } } state.completed(); }
Example #6
Source File: HoppingWindowExpressionTest.java From ksql-fork-with-deep-learning-function with Apache License 2.0 | 6 votes |
@Test public void shouldCreateHoppingWindowAggregate() { final KGroupedStream stream = EasyMock.createNiceMock(KGroupedStream.class); final TimeWindowedKStream windowedKStream = EasyMock.createNiceMock(TimeWindowedKStream.class); final UdafAggregator aggregator = EasyMock.createNiceMock(UdafAggregator.class); final HoppingWindowExpression windowExpression = new HoppingWindowExpression(10, TimeUnit.SECONDS, 4, TimeUnit.MILLISECONDS); final Initializer initializer = () -> 0; final Materialized<String, GenericRow, WindowStore<Bytes, byte[]>> store = Materialized.as("store"); EasyMock.expect(stream.windowedBy(TimeWindows.of(10000L).advanceBy(4L))).andReturn(windowedKStream); EasyMock.expect(windowedKStream.aggregate(same(initializer), same(aggregator), same(store))).andReturn(null); EasyMock.replay(stream, windowedKStream); windowExpression.applyAggregate(stream, initializer, aggregator, store); EasyMock.verify(stream, windowedKStream); }
Example #7
Source File: SingleRecordConsumerJob.java From java-11-examples with Apache License 2.0 | 6 votes |
@Override public ServiceResponse call() throws Exception { LOG.info("Consumer thread started."); while (true) { ConsumerRecords<String, Bytes> records = consumer.poll(Duration.ofMillis(10)); if (!records.isEmpty()) { for (ConsumerRecord<String, Bytes> record: records) { if (key.equals(record.key())) { LOG.info("Record: {}", record.key()); LOG.info("received response"); return dataMapper.deserialize(record.value(), ServiceResponse.class); } } } } }
Example #8
Source File: ProcessingServiceBackend.java From java-11-examples with Apache License 2.0 | 6 votes |
public void start() { Collection<String> topics = Collections.singletonList(TOPIC_SERVICE_REQUESTS); this.consumer.subscribe(topics); LOG.info("Waiting for requests {} ...", serviceId); this.running = true; while (running) { ConsumerRecords<String, Bytes> records = consumer.poll(Duration.ofMillis(10)); if (!records.isEmpty()) { for (ConsumerRecord<String, Bytes> record: records) { try { ServiceRequest request = dataMapper.deserialize(record.value(), ServiceRequest.class); LOG.info("Received Request: {}:{}:{}", record.key(), request.getClientId(), request.getTaskId()); ServiceResponse response = new ServiceResponse(request.getTaskId(), request.getClientId(), request.getData(), "response:" + request.getData()); Bytes bytes = dataMapper.serialize(response); ProducerRecord<String, Bytes> recordReply = new ProducerRecord<>(TOPIC_SERVICE_RESPONSES, response.getTaskId(), bytes); producer.send(recordReply); LOG.info("Response has been send !"); } catch (IOException e) { LOG.error("Exception: ", e); } } } } LOG.info("done {}.", serviceId); }
Example #9
Source File: StreamDemo.java From javatech with Creative Commons Attribution Share Alike 4.0 International | 6 votes |
public static void main(String[] args) { // 1. 指定流的配置 Properties config = new Properties(); config.put(StreamsConfig.APPLICATION_ID_CONFIG, "wordcount-application"); config.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, HOST); config.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass()); config.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass()); // 设置流构造器 StreamsBuilder builder = new StreamsBuilder(); KStream<String, String> textLines = builder.stream("TextLinesTopic"); KTable<String, Long> wordCounts = textLines .flatMapValues(textLine -> Arrays.asList(textLine.toLowerCase().split("\\W+"))) .groupBy((key, word) -> word) .count(Materialized.<String, Long, KeyValueStore<Bytes, byte[]>>as("counts-store")); wordCounts.toStream().to("WordsWithCountsTopic", Produced.with(Serdes.String(), Serdes.Long())); // 根据流构造器和流配置初始化 Kafka 流 KafkaStreams streams = new KafkaStreams(builder.build(), config); streams.start(); }
Example #10
Source File: TopicStreamWriterFormatTest.java From ksql-fork-with-deep-learning-function with Apache License 2.0 | 6 votes |
@Test public void shouldNotMatchAvroFormatter() throws Exception { /** * Setup expects */ SchemaRegistryClient schemaRegistryClient = mock(SchemaRegistryClient.class); replay(schemaRegistryClient); /** * Test data */ ConsumerRecord<String, Bytes> record = new ConsumerRecord<String, Bytes>("topic", 1, 1, "key", new Bytes("test-data".getBytes())); /** Assert */ assertFalse(TopicStreamWriter.Format.AVRO.isFormat("topic", record, schemaRegistryClient)); }
Example #11
Source File: TopicStreamWriterFormatTest.java From ksql-fork-with-deep-learning-function with Apache License 2.0 | 6 votes |
@Test public void shouldNotMatchJsonFormatter() throws Exception { SchemaRegistryClient schemaRegistryClient = mock(SchemaRegistryClient.class); replay(schemaRegistryClient); /** * Test data */ String json = "{ BAD DATA \"name\": \"myrecord\"," + " \"type\": \"record\"" + "}"; ConsumerRecord<String, Bytes> record = new ConsumerRecord<String, Bytes>("topic", 1, 1, "key", new Bytes(json.getBytes())); assertFalse(TopicStreamWriter.Format.JSON.isFormat("topic", record, schemaRegistryClient)); }
Example #12
Source File: KGraph.java From kafka-graphs with Apache License 2.0 | 6 votes |
public KGraph<K, VV, EV> subgraph(Predicate<K, VV> vertexFilter, Predicate<Edge<K>, EV> edgeFilter) { KTable<K, VV> filteredVertices = vertices.filter(vertexFilter); KTable<Edge<K>, EV> remainingEdges = edgesBySource() .join(filteredVertices, (e, v) -> e, Joined.with(keySerde(), new KryoSerde<>(), vertexValueSerde())) .map((k, edge) -> new KeyValue<>(edge.target(), edge)) .join(filteredVertices, (e, v) -> e, Joined.with(keySerde(), new KryoSerde<>(), vertexValueSerde())) .map((k, edge) -> new KeyValue<>(new Edge<>(edge.source(), edge.target()), edge.value())) .groupByKey(Grouped.with(new KryoSerde<>(), edgeValueSerde())) .reduce((v1, v2) -> v2, Materialized.with(new KryoSerde<>(), edgeValueSerde())); KTable<Edge<K>, EV> filteredEdges = remainingEdges .filter(edgeFilter, Materialized.<Edge<K>, EV, KeyValueStore<Bytes, byte[]>>as(generateStoreName()).withKeySerde(new KryoSerde<>()).withValueSerde(edgeValueSerde())); return new KGraph<>(filteredVertices, filteredEdges, serialized); }
Example #13
Source File: KGraph.java From kafka-graphs with Apache License 2.0 | 6 votes |
public <T> KGraph<K, VV, EV> joinWithEdgesOnTarget(KTable<K, T> inputDataSet, final EdgeJoinFunction<EV, T> edgeJoinFunction) { KTable<Edge<K>, EV> resultedEdges = edgesGroupedByTarget() .leftJoin(inputDataSet, new ApplyLeftJoinToEdgeValuesOnEitherSourceOrTarget<>(edgeJoinFunction), Materialized.with(keySerde(), new KryoSerde<>())) .toStream() .flatMap((k, edgeWithValues) -> { List<KeyValue<Edge<K>, EV>> edges = new ArrayList<>(); for (EdgeWithValue<K, EV> edge : edgeWithValues) { edges.add(new KeyValue<>(new Edge<>(edge.source(), edge.target()), edge.value())); } return edges; }) .groupByKey(Grouped.with(new KryoSerde<>(), edgeValueSerde())) .<EV>reduce((v1, v2) -> v2, Materialized.<Edge<K>, EV, KeyValueStore<Bytes, byte[]>>as( generateStoreName()).withKeySerde(new KryoSerde<>()).withValueSerde(edgeValueSerde())); return new KGraph<>(vertices, resultedEdges, serialized); }
Example #14
Source File: KafkaStreamsAggregateSample.java From spring-cloud-stream-samples with Apache License 2.0 | 6 votes |
@Bean public Consumer<KStream<String, DomainEvent>> aggregate() { ObjectMapper mapper = new ObjectMapper(); Serde<DomainEvent> domainEventSerde = new JsonSerde<>( DomainEvent.class, mapper ); return input -> input .groupBy( (s, domainEvent) -> domainEvent.boardUuid, Grouped.with(null, domainEventSerde)) .aggregate( String::new, (s, domainEvent, board) -> board.concat(domainEvent.eventType), Materialized.<String, String, KeyValueStore<Bytes, byte[]>>as("test-events-snapshots") .withKeySerde(Serdes.String()). withValueSerde(Serdes.String()) ); }
Example #15
Source File: KGraph.java From kafka-graphs with Apache License 2.0 | 6 votes |
public <T> KGraph<K, VV, EV> joinWithEdgesOnSource(KTable<K, T> inputDataSet, final EdgeJoinFunction<EV, T> edgeJoinFunction) { KTable<Edge<K>, EV> resultedEdges = edgesGroupedBySource() .leftJoin(inputDataSet, new ApplyLeftJoinToEdgeValuesOnEitherSourceOrTarget<>(edgeJoinFunction), Materialized.with(keySerde(), new KryoSerde<>())) .toStream() .flatMap((k, edgeWithValues) -> { List<KeyValue<Edge<K>, EV>> edges = new ArrayList<>(); for (EdgeWithValue<K, EV> edge : edgeWithValues) { edges.add(new KeyValue<>(new Edge<>(edge.source(), edge.target()), edge.value())); } return edges; }) .groupByKey(Grouped.with(new KryoSerde<>(), edgeValueSerde())) .<EV>reduce((v1, v2) -> v2, Materialized.<Edge<K>, EV, KeyValueStore<Bytes, byte[]>>as( generateStoreName()).withKeySerde(new KryoSerde<>()).withValueSerde(edgeValueSerde())); return new KGraph<>(this.vertices, resultedEdges, serialized); }
Example #16
Source File: KafkaStreamLevelConsumer.java From incubator-pinot with Apache License 2.0 | 5 votes |
@Override public GenericRow next(GenericRow destination) { if (kafkaIterator == null || !kafkaIterator.hasNext()) { updateKafkaIterator(); } if (kafkaIterator.hasNext()) { try { final ConsumerRecord<Bytes, Bytes> record = kafkaIterator.next(); updateOffsets(record.partition(), record.offset()); destination = _messageDecoder.decode(record.value().get(), destination); ++currentCount; final long now = System.currentTimeMillis(); // Log every minute or 100k events if (now - lastLogTime > 60000 || currentCount - lastCount >= 100000) { if (lastCount == 0) { INSTANCE_LOGGER.info("Consumed {} events from kafka stream {}", currentCount, _streamConfig.getTopicName()); } else { INSTANCE_LOGGER.info("Consumed {} events from kafka stream {} (rate:{}/s)", currentCount - lastCount, _streamConfig.getTopicName(), (float) (currentCount - lastCount) * 1000 / (now - lastLogTime)); } lastCount = currentCount; lastLogTime = now; } return destination; } catch (Exception e) { INSTANCE_LOGGER.warn("Caught exception while consuming events", e); throw e; } } return null; }
Example #17
Source File: KafkaStreamsInteractiveQueryApplication.java From spring-cloud-stream-samples with Apache License 2.0 | 5 votes |
@Bean public Function<KStream<Object, Product>, KStream<Integer, Long>> process() { return input -> input .filter((key, product) -> productIds().contains(product.getId())) .map((key, value) -> new KeyValue<>(value.id, value)) .groupByKey(Grouped.with(Serdes.Integer(), new JsonSerde<>(Product.class))) .count(Materialized.<Integer, Long, KeyValueStore<Bytes, byte[]>>as(STORE_NAME) .withKeySerde(Serdes.Integer()) .withValueSerde(Serdes.Long())) .toStream(); }
Example #18
Source File: RocksDBCacheTest.java From kcache with Apache License 2.0 | 5 votes |
@Test public void shouldPutOnlyIfAbsentValue() { RocksDBCache.init(); final Bytes keyBytes = new Bytes(stringSerializer.serialize(null, "one")); final byte[] valueBytes = stringSerializer.serialize(null, "A"); final byte[] valueBytesUpdate = stringSerializer.serialize(null, "B"); RocksDBCache.putIfAbsent(keyBytes, valueBytes); RocksDBCache.putIfAbsent(keyBytes, valueBytesUpdate); final String retrievedValue = stringDeserializer.deserialize(null, RocksDBCache.get(keyBytes)); assertEquals("A", retrievedValue); }
Example #19
Source File: WordCountProcessorApplicationTests.java From spring-cloud-stream-samples with Apache License 2.0 | 5 votes |
/** * Simple test validating count of one word */ @Test public void testOneWord() { final String nullKey = null; //Feed word "Hello" to inputTopic and no kafka key, timestamp is irrelevant in this case testDriver.pipeInput(recordFactory.create(INPUT_TOPIC, nullKey, "Hello", 1L)); //Read and validate output final ProducerRecord<Bytes, KafkaStreamsWordCountApplication.WordCount> output = readOutput(); assertThat(output).isNotNull(); assertThat(output.value()).isEqualToComparingFieldByField(new KafkaStreamsWordCountApplication.WordCount("hello", 1L, new Date(0), new Date(KafkaStreamsWordCountApplication.WordCountProcessorApplication.WINDOW_SIZE_MS))); //No more output in topic assertThat(readOutput()).isNull(); }
Example #20
Source File: KafkaStreamsWordCountApplication.java From spring-cloud-stream-samples with Apache License 2.0 | 5 votes |
@Bean public Function<KStream<Bytes, String>, KStream<Bytes, WordCount>> process() { return input -> input .flatMapValues(value -> Arrays.asList(value.toLowerCase().split("\\W+"))) .map((key, value) -> new KeyValue<>(value, value)) .groupByKey(Grouped.with(Serdes.String(), Serdes.String())) .windowedBy(TimeWindows.of(Duration.ofMillis(WINDOW_SIZE_MS))) .count(Materialized.as("WordCounts-1")) .toStream() .map((key, value) -> new KeyValue<>(null, new WordCount(key.key(), value, new Date(key.window().start()), new Date(key.window().end())))); }
Example #21
Source File: ProcessingServiceClient.java From java-11-examples with Apache License 2.0 | 5 votes |
@Override public Future<ServiceResponse> process(ServiceRequest request) throws ProcessingException { try { SingleRecordConsumerJob consumerJob = new SingleRecordConsumerJob(this.consumer, request.getTaskId()); Future<ServiceResponse> response = executor.submit(consumerJob); Bytes bytes = dataMapper.serialize(request); ProducerRecord<String, Bytes> record = new ProducerRecord<>(TOPIC_SERVICE_REQUESTS, request.getTaskId(), bytes); LOG.info("Sending request ..."); producer.send(record); return response; } catch (JsonProcessingException e) { throw new ProcessingException(e); } }
Example #22
Source File: CountVersionApplication.java From spring-cloud-stream-samples with Apache License 2.0 | 5 votes |
@Bean public Function<KStream<Object, Sensor>, KStream<String, Long>> process() { //The following Serde definitions are not needed in the topoloyy below //as we are not using it. However, if your topoloyg explicitly uses this //Serde, you need to configure this with the schema registry url as below. final Map<String, String> serdeConfig = Collections.singletonMap( AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, "http://localhost:8081"); final SpecificAvroSerde<Sensor> sensorSerde = new SpecificAvroSerde<>(); sensorSerde.configure(serdeConfig, false); return input -> input .map((k, value) -> { String newKey = "v1"; if (value.getId().toString().endsWith("v2")) { newKey = "v2"; } return new KeyValue<>(newKey, value); }) .groupByKey() .count(Materialized.<String, Long, KeyValueStore<Bytes, byte[]>>as(STORE_NAME) .withKeySerde(Serdes.String()) .withValueSerde(Serdes.Long())) .toStream(); }
Example #23
Source File: DomainEventSinkImpl.java From event-store-demo with GNU General Public License v3.0 | 5 votes |
@StreamListener( "input" ) public void process( KStream<Object, byte[]> input ) { log.debug( "process : enter" ); input .map( (key, value) -> { try { DomainEvent domainEvent = mapper.readValue( value, DomainEvent.class ); log.debug( "process : domainEvent=" + domainEvent ); return new KeyValue<>( domainEvent.getBoardUuid().toString(), domainEvent ); } catch( IOException e ) { log.error( "process : error converting json to DomainEvent", e ); } return null; }) .groupBy( (s, domainEvent) -> s, Serialized.with( Serdes.String(), domainEventSerde ) ) .aggregate( Board::new, (key, domainEvent, board) -> board.handleEvent( domainEvent ), Materialized.<String, Board, KeyValueStore<Bytes, byte[]>>as( BOARD_EVENTS_SNAPSHOTS ) .withKeySerde( Serdes.String() ) .withValueSerde( boardSerde ) ); log.debug( "process : exit" ); }
Example #24
Source File: KafkaPartitionLevelConsumer.java From incubator-pinot with Apache License 2.0 | 5 votes |
public MessageBatch fetchMessages(long startOffset, long endOffset, int timeoutMillis) throws TimeoutException { _consumer.seek(_topicPartition, startOffset); ConsumerRecords<String, Bytes> consumerRecords = _consumer.poll(Duration.ofMillis(timeoutMillis)); final Iterable<ConsumerRecord<String, Bytes>> messageAndOffsetIterable = buildOffsetFilteringIterable(consumerRecords.records(_topicPartition), startOffset, endOffset); return new KafkaMessageBatch(messageAndOffsetIterable); }
Example #25
Source File: WordCountProcessorApplicationTests.java From spring-cloud-stream-samples with Apache License 2.0 | 5 votes |
/** * Read counts from output to map ignoring start and end dates * If existing word is incremented, it can appear twice in output and is replaced in map * * @return Map of Word and counts */ private Map<String, Long> getOutputList() { final Map<String, Long> output = new HashMap<>(); ProducerRecord<Bytes, KafkaStreamsWordCountApplication.WordCount> outputRow; while ((outputRow = readOutput()) != null) { output.put(outputRow.value().getWord(), outputRow.value().getCount()); } return output; }
Example #26
Source File: SummaryBulkAggregation.java From kafka-graphs with Apache License 2.0 | 5 votes |
@SuppressWarnings("unchecked") @Override public KTable<Windowed<Short>, T> run(final KStream<Edge<K>, EV> edgeStream) { //For parallel window support we key the edge stream by partition and apply a parallel fold per partition. //Finally, we merge all locally combined results into our final graph aggregation property. KTable<Windowed<Short>, S> partialAgg = edgeStream .groupByKey(Grouped.with(new KryoSerde<>(), new KryoSerde<>())) .windowedBy(TimeWindows.of(Duration.ofMillis(timeMillis))) .aggregate(this::initialValue, new PartialAgg<>(updateFun())) .toStream() .groupBy((k, v) -> GLOBAL_KEY) .windowedBy(TimeWindows.of(Duration.ofMillis(timeMillis))) .reduce(combineFun()) .mapValues(aggregator(edgeStream), Materialized.<Windowed<Short>, S, KeyValueStore<Bytes, byte[]>> as(KGraph.generateStoreName()).withKeySerde(new KryoSerde<>()).withValueSerde(new KryoSerde<>())); if (transform() != null) { return partialAgg.mapValues( transform(), Materialized.<Windowed<Short>, T, KeyValueStore<Bytes, byte[]>> as(KGraph.generateStoreName()).withKeySerde(new KryoSerde<>()).withValueSerde(new KryoSerde<>()) ); } return (KTable<Windowed<Short>, T>) partialAgg; }
Example #27
Source File: KGraph.java From kafka-graphs with Apache License 2.0 | 5 votes |
public KGraph<K, VV, EV> undirected() { KTable<Edge<K>, EV> undirectedEdges = edges .toStream() .flatMap(new UndirectEdges<>()) .groupByKey(Grouped.with(new KryoSerde<>(), serialized.edgeValueSerde())) .reduce((v1, v2) -> v2, Materialized.<Edge<K>, EV, KeyValueStore<Bytes, byte[]>>as(generateStoreName()) .withKeySerde(new KryoSerde<>()).withValueSerde(serialized.edgeValueSerde())); return new KGraph<>(vertices, undirectedEdges, serialized); }
Example #28
Source File: KGraph.java From kafka-graphs with Apache License 2.0 | 5 votes |
public KGraph<K, VV, EV> filterOnVertices(Predicate<K, VV> vertexFilter) { KTable<K, VV> filteredVertices = vertices.filter(vertexFilter); KTable<Edge<K>, EV> remainingEdges = edgesBySource() .join(filteredVertices, (e, v) -> e, Joined.with(keySerde(), new KryoSerde<>(), vertexValueSerde())) .map((k, edge) -> new KeyValue<>(edge.target(), edge)) .join(filteredVertices, (e, v) -> e, Joined.with(keySerde(), new KryoSerde<>(), vertexValueSerde())) .map((k, edge) -> new KeyValue<>(new Edge<>(edge.source(), edge.target()), edge.value())) .groupByKey(Grouped.with(new KryoSerde<>(), edgeValueSerde())) .reduce((v1, v2) -> v2, Materialized.<Edge<K>, EV, KeyValueStore<Bytes, byte[]>>as(generateStoreName()).withKeySerde(new KryoSerde<>()).withValueSerde(edgeValueSerde())); return new KGraph<>(filteredVertices, remainingEdges, serialized); }
Example #29
Source File: DomainEventSinkImpl.java From event-store-demo with GNU General Public License v3.0 | 5 votes |
@StreamListener( "input" ) public void process( KStream<Object, byte[]> input ) { log.debug( "process : enter" ); input .map( (key, value) -> { try { DomainEvent domainEvent = mapper.readValue( value, DomainEvent.class ); log.debug( "process : domainEvent=" + domainEvent ); return new KeyValue<>( domainEvent.getBoardUuid().toString(), domainEvent ); } catch( IOException e ) { log.error( "process : error converting json to DomainEvent", e ); } return null; }) .groupBy( (s, domainEvent) -> s, Serialized.with( Serdes.String(), domainEventSerde ) ) .aggregate( Board::new, (key, domainEvent, board) -> board.handleEvent( domainEvent ), Materialized.<String, Board, KeyValueStore<Bytes, byte[]>>as( BOARD_EVENTS_SNAPSHOTS ) .withKeySerde( Serdes.String() ) .withValueSerde( boardSerde ) ); log.debug( "process : exit" ); }
Example #30
Source File: KafkaPartitionLevelConsumer.java From incubator-pinot with Apache License 2.0 | 5 votes |
private Iterable<ConsumerRecord<String, Bytes>> buildOffsetFilteringIterable( final List<ConsumerRecord<String, Bytes>> messageAndOffsets, final long startOffset, final long endOffset) { return Iterables.filter(messageAndOffsets, input -> { // Filter messages that are either null or have an offset ∉ [startOffset, endOffset] return input != null && input.offset() >= startOffset && (endOffset > input.offset() || endOffset == -1); }); }