org.apache.kafka.common.utils.Bytes Java Examples

The following examples show how to use org.apache.kafka.common.utils.Bytes. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ScsApplication.java    From spring_io_2019 with Apache License 2.0 7 votes vote down vote up
@StreamListener
@SendTo(Bindings.RATED_MOVIES)
KStream<Long, RatedMovie> rateMoviesFor(@Input(Bindings.AVG_TABLE) KTable<Long, Double> ratings,
                                        @Input(Bindings.MOVIES) KTable<Long, Movie> movies) {

  ValueJoiner<Movie, Double, RatedMovie> joiner = (movie, rating) ->
      new RatedMovie(movie.getMovieId(), movie.getReleaseYear(), movie.getTitle(), rating);

  movies
      .join(ratings, joiner, Materialized
          .<Long, RatedMovie, KeyValueStore<Bytes, byte[]>>as(Bindings.RATED_MOVIES_STORE)
          .withKeySerde(Serdes.Long())
          .withValueSerde(new JsonSerde<>(RatedMovie.class)));

  return movies.join(ratings, joiner).toStream();
}
 
Example #2
Source File: TopicStreamWriterFormatTest.java    From ksql-fork-with-deep-learning-function with Apache License 2.0 6 votes vote down vote up
@Test
public void shouldMatchJsonFormatter() throws Exception {

  SchemaRegistryClient schemaRegistryClient = mock(SchemaRegistryClient.class);
  replay(schemaRegistryClient);

  /**
   * Test data
   */
  String json = "{    \"name\": \"myrecord\"," +
          "    \"type\": \"record\"" +
          "}";

  ConsumerRecord<String, Bytes> record = new ConsumerRecord<String, Bytes>("topic", 1, 1, "key", new Bytes(json.getBytes()));

  assertTrue(TopicStreamWriter.Format.JSON.isFormat("topic", record, schemaRegistryClient));
}
 
Example #3
Source File: TumblingWindowExpressionTest.java    From ksql-fork-with-deep-learning-function with Apache License 2.0 6 votes vote down vote up
@Test
public void shouldCreateTumblingWindowAggregate() {
  final KGroupedStream stream = EasyMock.createNiceMock(KGroupedStream.class);
  final TimeWindowedKStream windowedKStream = EasyMock.createNiceMock(TimeWindowedKStream.class);
  final UdafAggregator aggregator = EasyMock.createNiceMock(UdafAggregator.class);
  final TumblingWindowExpression windowExpression = new TumblingWindowExpression(10, TimeUnit.SECONDS);
  final Initializer initializer = () -> 0;
  final Materialized<String, GenericRow, WindowStore<Bytes, byte[]>> store = Materialized.as("store");

  EasyMock.expect(stream.windowedBy(TimeWindows.of(10000L))).andReturn(windowedKStream);
  EasyMock.expect(windowedKStream.aggregate(same(initializer), same(aggregator), same(store))).andReturn(null);
  EasyMock.replay(stream, windowedKStream);

  windowExpression.applyAggregate(stream, initializer, aggregator, store);
  EasyMock.verify(stream, windowedKStream);
}
 
Example #4
Source File: KafkaMetricReceiver.java    From super-cloudops with Apache License 2.0 6 votes vote down vote up
/**
 * Receiving consumer messages on multiple topics
 *
 * @param records
 * @param ack
 */
@KafkaListener(topicPattern = TOPIC_KAFKA_RECEIVE_PATTERN, containerFactory = BEAN_KAFKA_BATCH_FACTORY)
public void onMetricReceive(List<ConsumerRecord<byte[], Bytes>> records, Acknowledgment ack) {
	try {
		if (log.isDebugEnabled()) {
			log.debug("Receive metric records - {}", records);
		}
		if (log.isInfoEnabled()) {
			log.info("Receive metric records size - {}", records.size());
		}

		doProcess(records, new MultiAcknowledgmentState(ack));
	} catch (Exception e) {
		log.error(String.format("Failed to receive process for ", records.size()), e);
	}
}
 
Example #5
Source File: KafkaMetricReceiver.java    From super-cloudops with Apache License 2.0 6 votes vote down vote up
/**
 * UMC agent metric processing.
 *
 * @param records
 * @param state
 */
private void doProcess(List<ConsumerRecord<byte[], Bytes>> records, MultiAcknowledgmentState state) {
	for (ConsumerRecord<byte[], Bytes> record : records) {
		try {
			MetricAggregate aggregate = MetricAggregate.parseFrom(record.value().get());
			if (log.isDebugEnabled()) {
				log.debug("Put metric aggregate for - {}", aggregate);
			}

			// Storage metrics.
			putMetrics(aggregate);

			// Metrics alarm.
			alarm(aggregate);
		} catch (InvalidProtocolBufferException e) {
			log.error("Failed to parse metric message.", e);
		}
	}
	state.completed();
}
 
Example #6
Source File: HoppingWindowExpressionTest.java    From ksql-fork-with-deep-learning-function with Apache License 2.0 6 votes vote down vote up
@Test
public void shouldCreateHoppingWindowAggregate() {
  final KGroupedStream stream = EasyMock.createNiceMock(KGroupedStream.class);
  final TimeWindowedKStream windowedKStream = EasyMock.createNiceMock(TimeWindowedKStream.class);
  final UdafAggregator aggregator = EasyMock.createNiceMock(UdafAggregator.class);
  final HoppingWindowExpression windowExpression = new HoppingWindowExpression(10, TimeUnit.SECONDS, 4, TimeUnit.MILLISECONDS);
  final Initializer initializer = () -> 0;
  final Materialized<String, GenericRow, WindowStore<Bytes, byte[]>> store = Materialized.as("store");

  EasyMock.expect(stream.windowedBy(TimeWindows.of(10000L).advanceBy(4L))).andReturn(windowedKStream);
  EasyMock.expect(windowedKStream.aggregate(same(initializer), same(aggregator), same(store))).andReturn(null);
  EasyMock.replay(stream, windowedKStream);

  windowExpression.applyAggregate(stream, initializer, aggregator, store);
  EasyMock.verify(stream, windowedKStream);
}
 
Example #7
Source File: SingleRecordConsumerJob.java    From java-11-examples with Apache License 2.0 6 votes vote down vote up
@Override
public ServiceResponse call() throws Exception {
    LOG.info("Consumer thread started.");
    while (true) {
        ConsumerRecords<String, Bytes> records = consumer.poll(Duration.ofMillis(10));
        if (!records.isEmpty()) {
            for (ConsumerRecord<String, Bytes> record: records) {
                if (key.equals(record.key())) {
                    LOG.info("Record: {}", record.key());
                    LOG.info("received response");
                    return dataMapper.deserialize(record.value(), ServiceResponse.class);
                }
            }
        }
    }
}
 
Example #8
Source File: ProcessingServiceBackend.java    From java-11-examples with Apache License 2.0 6 votes vote down vote up
public void start() {
    Collection<String> topics = Collections.singletonList(TOPIC_SERVICE_REQUESTS);
    this.consumer.subscribe(topics);
    LOG.info("Waiting for requests {} ...", serviceId);
    this.running = true;
    while (running) {
        ConsumerRecords<String, Bytes> records = consumer.poll(Duration.ofMillis(10));
        if (!records.isEmpty()) {
            for (ConsumerRecord<String, Bytes> record: records) {
                try {
                    ServiceRequest request = dataMapper.deserialize(record.value(), ServiceRequest.class);
                    LOG.info("Received Request: {}:{}:{}", record.key(), request.getClientId(), request.getTaskId());
                    ServiceResponse response =
                            new ServiceResponse(request.getTaskId(), request.getClientId(), request.getData(), "response:" + request.getData());
                    Bytes bytes = dataMapper.serialize(response);
                    ProducerRecord<String, Bytes> recordReply = new ProducerRecord<>(TOPIC_SERVICE_RESPONSES, response.getTaskId(), bytes);
                    producer.send(recordReply);
                    LOG.info("Response has been send !");
                } catch (IOException e) {
                    LOG.error("Exception: ", e);
                }
            }
        }
    }
    LOG.info("done {}.", serviceId);
}
 
Example #9
Source File: StreamDemo.java    From javatech with Creative Commons Attribution Share Alike 4.0 International 6 votes vote down vote up
public static void main(String[] args) {
	// 1. 指定流的配置
	Properties config = new Properties();
	config.put(StreamsConfig.APPLICATION_ID_CONFIG, "wordcount-application");
	config.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, HOST);
	config.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
	config.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass());

	// 设置流构造器
	StreamsBuilder builder = new StreamsBuilder();
	KStream<String, String> textLines = builder.stream("TextLinesTopic");
	KTable<String, Long> wordCounts = textLines
		.flatMapValues(textLine -> Arrays.asList(textLine.toLowerCase().split("\\W+")))
		.groupBy((key, word) -> word)
		.count(Materialized.<String, Long, KeyValueStore<Bytes, byte[]>>as("counts-store"));
	wordCounts.toStream().to("WordsWithCountsTopic", Produced.with(Serdes.String(), Serdes.Long()));

	// 根据流构造器和流配置初始化 Kafka 流
	KafkaStreams streams = new KafkaStreams(builder.build(), config);
	streams.start();
}
 
Example #10
Source File: TopicStreamWriterFormatTest.java    From ksql-fork-with-deep-learning-function with Apache License 2.0 6 votes vote down vote up
@Test
public void shouldNotMatchAvroFormatter() throws Exception {

  /**
   * Setup expects
   */
  SchemaRegistryClient schemaRegistryClient = mock(SchemaRegistryClient.class);
  replay(schemaRegistryClient);

  /**
   * Test data
   */
  ConsumerRecord<String, Bytes> record = new ConsumerRecord<String, Bytes>("topic", 1, 1, "key", new Bytes("test-data".getBytes()));

  /** Assert
   */
  assertFalse(TopicStreamWriter.Format.AVRO.isFormat("topic", record, schemaRegistryClient));
}
 
Example #11
Source File: TopicStreamWriterFormatTest.java    From ksql-fork-with-deep-learning-function with Apache License 2.0 6 votes vote down vote up
@Test
public void shouldNotMatchJsonFormatter() throws Exception {

  SchemaRegistryClient schemaRegistryClient = mock(SchemaRegistryClient.class);
  replay(schemaRegistryClient);

  /**
   * Test data
   */
  String json = "{  BAD DATA  \"name\": \"myrecord\"," +
          "    \"type\": \"record\"" +
          "}";

  ConsumerRecord<String, Bytes> record = new ConsumerRecord<String, Bytes>("topic", 1, 1, "key", new Bytes(json.getBytes()));


  assertFalse(TopicStreamWriter.Format.JSON.isFormat("topic", record, schemaRegistryClient));
}
 
Example #12
Source File: KGraph.java    From kafka-graphs with Apache License 2.0 6 votes vote down vote up
public KGraph<K, VV, EV> subgraph(Predicate<K, VV> vertexFilter, Predicate<Edge<K>, EV> edgeFilter) {
    KTable<K, VV> filteredVertices = vertices.filter(vertexFilter);

    KTable<Edge<K>, EV> remainingEdges = edgesBySource()
        .join(filteredVertices, (e, v) -> e, Joined.with(keySerde(), new KryoSerde<>(), vertexValueSerde()))
        .map((k, edge) -> new KeyValue<>(edge.target(), edge))
        .join(filteredVertices, (e, v) -> e, Joined.with(keySerde(), new KryoSerde<>(), vertexValueSerde()))
        .map((k, edge) -> new KeyValue<>(new Edge<>(edge.source(), edge.target()), edge.value()))
        .groupByKey(Grouped.with(new KryoSerde<>(), edgeValueSerde()))
        .reduce((v1, v2) -> v2, Materialized.with(new KryoSerde<>(), edgeValueSerde()));

    KTable<Edge<K>, EV> filteredEdges = remainingEdges
        .filter(edgeFilter, Materialized.<Edge<K>, EV, KeyValueStore<Bytes, byte[]>>as(generateStoreName()).withKeySerde(new KryoSerde<>()).withValueSerde(edgeValueSerde()));

    return new KGraph<>(filteredVertices, filteredEdges, serialized);
}
 
Example #13
Source File: KGraph.java    From kafka-graphs with Apache License 2.0 6 votes vote down vote up
public <T> KGraph<K, VV, EV> joinWithEdgesOnTarget(KTable<K, T> inputDataSet,
                                                   final EdgeJoinFunction<EV, T> edgeJoinFunction) {

    KTable<Edge<K>, EV> resultedEdges = edgesGroupedByTarget()
        .leftJoin(inputDataSet,
            new ApplyLeftJoinToEdgeValuesOnEitherSourceOrTarget<>(edgeJoinFunction),
            Materialized.with(keySerde(), new KryoSerde<>()))
        .toStream()
        .flatMap((k, edgeWithValues) -> {
            List<KeyValue<Edge<K>, EV>> edges = new ArrayList<>();
            for (EdgeWithValue<K, EV> edge : edgeWithValues) {
                edges.add(new KeyValue<>(new Edge<>(edge.source(), edge.target()), edge.value()));
            }
            return edges;
        })
        .groupByKey(Grouped.with(new KryoSerde<>(), edgeValueSerde()))
        .<EV>reduce((v1, v2) -> v2, Materialized.<Edge<K>, EV, KeyValueStore<Bytes, byte[]>>as(
            generateStoreName()).withKeySerde(new KryoSerde<>()).withValueSerde(edgeValueSerde()));

    return new KGraph<>(vertices, resultedEdges, serialized);
}
 
Example #14
Source File: KafkaStreamsAggregateSample.java    From spring-cloud-stream-samples with Apache License 2.0 6 votes vote down vote up
@Bean
public Consumer<KStream<String, DomainEvent>> aggregate() {

	ObjectMapper mapper = new ObjectMapper();
	Serde<DomainEvent> domainEventSerde = new JsonSerde<>( DomainEvent.class, mapper );

	return input -> input
			.groupBy(
					(s, domainEvent) -> domainEvent.boardUuid,
					Grouped.with(null, domainEventSerde))
			.aggregate(
					String::new,
					(s, domainEvent, board) -> board.concat(domainEvent.eventType),
					Materialized.<String, String, KeyValueStore<Bytes, byte[]>>as("test-events-snapshots")
							.withKeySerde(Serdes.String()).
							withValueSerde(Serdes.String())
			);
}
 
Example #15
Source File: KGraph.java    From kafka-graphs with Apache License 2.0 6 votes vote down vote up
public <T> KGraph<K, VV, EV> joinWithEdgesOnSource(KTable<K, T> inputDataSet,
                                                   final EdgeJoinFunction<EV, T> edgeJoinFunction) {

    KTable<Edge<K>, EV> resultedEdges = edgesGroupedBySource()
        .leftJoin(inputDataSet,
            new ApplyLeftJoinToEdgeValuesOnEitherSourceOrTarget<>(edgeJoinFunction),
            Materialized.with(keySerde(), new KryoSerde<>()))
        .toStream()
        .flatMap((k, edgeWithValues) -> {
            List<KeyValue<Edge<K>, EV>> edges = new ArrayList<>();
            for (EdgeWithValue<K, EV> edge : edgeWithValues) {
                edges.add(new KeyValue<>(new Edge<>(edge.source(), edge.target()), edge.value()));
            }
            return edges;
        })
        .groupByKey(Grouped.with(new KryoSerde<>(), edgeValueSerde()))
        .<EV>reduce((v1, v2) -> v2, Materialized.<Edge<K>, EV, KeyValueStore<Bytes, byte[]>>as(
            generateStoreName()).withKeySerde(new KryoSerde<>()).withValueSerde(edgeValueSerde()));

    return new KGraph<>(this.vertices, resultedEdges, serialized);
}
 
Example #16
Source File: KafkaStreamLevelConsumer.java    From incubator-pinot with Apache License 2.0 5 votes vote down vote up
@Override
public GenericRow next(GenericRow destination) {
  if (kafkaIterator == null || !kafkaIterator.hasNext()) {
    updateKafkaIterator();
  }
  if (kafkaIterator.hasNext()) {
    try {
      final ConsumerRecord<Bytes, Bytes> record = kafkaIterator.next();
      updateOffsets(record.partition(), record.offset());
      destination = _messageDecoder.decode(record.value().get(), destination);

      ++currentCount;

      final long now = System.currentTimeMillis();
      // Log every minute or 100k events
      if (now - lastLogTime > 60000 || currentCount - lastCount >= 100000) {
        if (lastCount == 0) {
          INSTANCE_LOGGER.info("Consumed {} events from kafka stream {}", currentCount, _streamConfig.getTopicName());
        } else {
          INSTANCE_LOGGER.info("Consumed {} events from kafka stream {} (rate:{}/s)", currentCount - lastCount,
              _streamConfig.getTopicName(), (float) (currentCount - lastCount) * 1000 / (now - lastLogTime));
        }
        lastCount = currentCount;
        lastLogTime = now;
      }
      return destination;
    } catch (Exception e) {
      INSTANCE_LOGGER.warn("Caught exception while consuming events", e);
      throw e;
    }
  }
  return null;
}
 
Example #17
Source File: KafkaStreamsInteractiveQueryApplication.java    From spring-cloud-stream-samples with Apache License 2.0 5 votes vote down vote up
@Bean
public Function<KStream<Object, Product>, KStream<Integer, Long>> process() {

	return input -> input
			.filter((key, product) -> productIds().contains(product.getId()))
			.map((key, value) -> new KeyValue<>(value.id, value))
			.groupByKey(Grouped.with(Serdes.Integer(), new JsonSerde<>(Product.class)))
			.count(Materialized.<Integer, Long, KeyValueStore<Bytes, byte[]>>as(STORE_NAME)
				.withKeySerde(Serdes.Integer())
				.withValueSerde(Serdes.Long()))
			.toStream();
}
 
Example #18
Source File: RocksDBCacheTest.java    From kcache with Apache License 2.0 5 votes vote down vote up
@Test
public void shouldPutOnlyIfAbsentValue() {
    RocksDBCache.init();
    final Bytes keyBytes = new Bytes(stringSerializer.serialize(null, "one"));
    final byte[] valueBytes = stringSerializer.serialize(null, "A");
    final byte[] valueBytesUpdate = stringSerializer.serialize(null, "B");

    RocksDBCache.putIfAbsent(keyBytes, valueBytes);
    RocksDBCache.putIfAbsent(keyBytes, valueBytesUpdate);

    final String retrievedValue = stringDeserializer.deserialize(null, RocksDBCache.get(keyBytes));
    assertEquals("A", retrievedValue);
}
 
Example #19
Source File: WordCountProcessorApplicationTests.java    From spring-cloud-stream-samples with Apache License 2.0 5 votes vote down vote up
/**
 * Simple test validating count of one word
 */
@Test
public void testOneWord() {
    final String nullKey = null;
    //Feed word "Hello" to inputTopic and no kafka key, timestamp is irrelevant in this case
    testDriver.pipeInput(recordFactory.create(INPUT_TOPIC, nullKey, "Hello", 1L));
    //Read and validate output
    final ProducerRecord<Bytes, KafkaStreamsWordCountApplication.WordCount> output = readOutput();
    assertThat(output).isNotNull();
    assertThat(output.value()).isEqualToComparingFieldByField(new KafkaStreamsWordCountApplication.WordCount("hello", 1L, new Date(0), new Date(KafkaStreamsWordCountApplication.WordCountProcessorApplication.WINDOW_SIZE_MS)));
    //No more output in topic
    assertThat(readOutput()).isNull();
}
 
Example #20
Source File: KafkaStreamsWordCountApplication.java    From spring-cloud-stream-samples with Apache License 2.0 5 votes vote down vote up
@Bean
public Function<KStream<Bytes, String>, KStream<Bytes, WordCount>> process() {

	return input -> input
			.flatMapValues(value -> Arrays.asList(value.toLowerCase().split("\\W+")))
			.map((key, value) -> new KeyValue<>(value, value))
			.groupByKey(Grouped.with(Serdes.String(), Serdes.String()))
			.windowedBy(TimeWindows.of(Duration.ofMillis(WINDOW_SIZE_MS)))
			.count(Materialized.as("WordCounts-1"))
			.toStream()
			.map((key, value) -> new KeyValue<>(null, new WordCount(key.key(), value, new Date(key.window().start()), new Date(key.window().end()))));
}
 
Example #21
Source File: ProcessingServiceClient.java    From java-11-examples with Apache License 2.0 5 votes vote down vote up
@Override
public Future<ServiceResponse> process(ServiceRequest request) throws ProcessingException {
    try {
        SingleRecordConsumerJob consumerJob = new SingleRecordConsumerJob(this.consumer, request.getTaskId());
        Future<ServiceResponse> response = executor.submit(consumerJob);
        Bytes bytes = dataMapper.serialize(request);
        ProducerRecord<String, Bytes> record = new ProducerRecord<>(TOPIC_SERVICE_REQUESTS, request.getTaskId(), bytes);
        LOG.info("Sending request ...");
        producer.send(record);
        return response;
    } catch (JsonProcessingException e) {
        throw new ProcessingException(e);
    }
}
 
Example #22
Source File: CountVersionApplication.java    From spring-cloud-stream-samples with Apache License 2.0 5 votes vote down vote up
@Bean
public Function<KStream<Object, Sensor>, KStream<String, Long>> process() {

	//The following Serde definitions are not needed in the topoloyy below
	//as we are not using it. However, if your topoloyg explicitly uses this
	//Serde, you need to configure this with the schema registry url as below.

	final Map<String, String> serdeConfig = Collections.singletonMap(
			AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, "http://localhost:8081");

	final SpecificAvroSerde<Sensor> sensorSerde = new SpecificAvroSerde<>();
	sensorSerde.configure(serdeConfig, false);

	return input -> input
			.map((k, value) -> {
				String newKey = "v1";
				if (value.getId().toString().endsWith("v2")) {
					newKey = "v2";
				}
				return new KeyValue<>(newKey, value);
			})
			.groupByKey()
			.count(Materialized.<String, Long, KeyValueStore<Bytes, byte[]>>as(STORE_NAME)
					.withKeySerde(Serdes.String())
					.withValueSerde(Serdes.Long()))
			.toStream();

}
 
Example #23
Source File: DomainEventSinkImpl.java    From event-store-demo with GNU General Public License v3.0 5 votes vote down vote up
@StreamListener( "input" )
public void process( KStream<Object, byte[]> input ) {
    log.debug( "process : enter" );

    input
            .map( (key, value) -> {

                try {

                    DomainEvent domainEvent = mapper.readValue( value, DomainEvent.class );
                    log.debug( "process : domainEvent=" + domainEvent );

                    return new KeyValue<>( domainEvent.getBoardUuid().toString(), domainEvent );

                } catch( IOException e ) {
                    log.error( "process : error converting json to DomainEvent", e );
                }

                return null;
            })
            .groupBy( (s, domainEvent) -> s, Serialized.with( Serdes.String(), domainEventSerde ) )
            .aggregate(
                    Board::new,
                    (key, domainEvent, board) -> board.handleEvent( domainEvent ),
                    Materialized.<String, Board, KeyValueStore<Bytes, byte[]>>as( BOARD_EVENTS_SNAPSHOTS )
                        .withKeySerde( Serdes.String() )
                        .withValueSerde( boardSerde )
            );

    log.debug( "process : exit" );
}
 
Example #24
Source File: KafkaPartitionLevelConsumer.java    From incubator-pinot with Apache License 2.0 5 votes vote down vote up
public MessageBatch fetchMessages(long startOffset, long endOffset, int timeoutMillis)
    throws TimeoutException {
  _consumer.seek(_topicPartition, startOffset);
  ConsumerRecords<String, Bytes> consumerRecords = _consumer.poll(Duration.ofMillis(timeoutMillis));
  final Iterable<ConsumerRecord<String, Bytes>> messageAndOffsetIterable =
      buildOffsetFilteringIterable(consumerRecords.records(_topicPartition), startOffset, endOffset);
  return new KafkaMessageBatch(messageAndOffsetIterable);
}
 
Example #25
Source File: WordCountProcessorApplicationTests.java    From spring-cloud-stream-samples with Apache License 2.0 5 votes vote down vote up
/**
 * Read counts from output to map ignoring start and end dates
 * If existing word is incremented, it can appear twice in output and is replaced in map
 *
 * @return Map of Word and counts
 */
private Map<String, Long> getOutputList() {
    final Map<String, Long> output = new HashMap<>();
    ProducerRecord<Bytes, KafkaStreamsWordCountApplication.WordCount> outputRow;
    while ((outputRow = readOutput()) != null) {
        output.put(outputRow.value().getWord(), outputRow.value().getCount());
    }
    return output;
}
 
Example #26
Source File: SummaryBulkAggregation.java    From kafka-graphs with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public KTable<Windowed<Short>, T> run(final KStream<Edge<K>, EV> edgeStream) {

    //For parallel window support we key the edge stream by partition and apply a parallel fold per partition.
    //Finally, we merge all locally combined results into our final graph aggregation property.
    KTable<Windowed<Short>, S> partialAgg = edgeStream
        .groupByKey(Grouped.with(new KryoSerde<>(), new KryoSerde<>()))
        .windowedBy(TimeWindows.of(Duration.ofMillis(timeMillis)))
        .aggregate(this::initialValue, new PartialAgg<>(updateFun()))
        .toStream()
        .groupBy((k, v) -> GLOBAL_KEY)
        .windowedBy(TimeWindows.of(Duration.ofMillis(timeMillis)))
        .reduce(combineFun())
        .mapValues(aggregator(edgeStream), Materialized.<Windowed<Short>, S, KeyValueStore<Bytes, byte[]>>
            as(KGraph.generateStoreName()).withKeySerde(new KryoSerde<>()).withValueSerde(new KryoSerde<>()));

    if (transform() != null) {
        return partialAgg.mapValues(
            transform(),
            Materialized.<Windowed<Short>, T, KeyValueStore<Bytes, byte[]>>
                as(KGraph.generateStoreName()).withKeySerde(new KryoSerde<>()).withValueSerde(new KryoSerde<>())
        );
    }

    return (KTable<Windowed<Short>, T>) partialAgg;
}
 
Example #27
Source File: KGraph.java    From kafka-graphs with Apache License 2.0 5 votes vote down vote up
public KGraph<K, VV, EV> undirected() {

        KTable<Edge<K>, EV> undirectedEdges = edges
            .toStream()
            .flatMap(new UndirectEdges<>())
            .groupByKey(Grouped.with(new KryoSerde<>(), serialized.edgeValueSerde()))
            .reduce((v1, v2) -> v2, Materialized.<Edge<K>, EV, KeyValueStore<Bytes, byte[]>>as(generateStoreName())
                .withKeySerde(new KryoSerde<>()).withValueSerde(serialized.edgeValueSerde()));

        return new KGraph<>(vertices, undirectedEdges, serialized);
    }
 
Example #28
Source File: KGraph.java    From kafka-graphs with Apache License 2.0 5 votes vote down vote up
public KGraph<K, VV, EV> filterOnVertices(Predicate<K, VV> vertexFilter) {
    KTable<K, VV> filteredVertices = vertices.filter(vertexFilter);

    KTable<Edge<K>, EV> remainingEdges = edgesBySource()
        .join(filteredVertices, (e, v) -> e, Joined.with(keySerde(), new KryoSerde<>(), vertexValueSerde()))
        .map((k, edge) -> new KeyValue<>(edge.target(), edge))
        .join(filteredVertices, (e, v) -> e, Joined.with(keySerde(), new KryoSerde<>(), vertexValueSerde()))
        .map((k, edge) -> new KeyValue<>(new Edge<>(edge.source(), edge.target()), edge.value()))
        .groupByKey(Grouped.with(new KryoSerde<>(), edgeValueSerde()))
        .reduce((v1, v2) -> v2, Materialized.<Edge<K>, EV, KeyValueStore<Bytes, byte[]>>as(generateStoreName()).withKeySerde(new KryoSerde<>()).withValueSerde(edgeValueSerde()));

    return new KGraph<>(filteredVertices, remainingEdges, serialized);
}
 
Example #29
Source File: DomainEventSinkImpl.java    From event-store-demo with GNU General Public License v3.0 5 votes vote down vote up
@StreamListener( "input" )
public void process( KStream<Object, byte[]> input ) {
    log.debug( "process : enter" );

    input
            .map( (key, value) -> {

                try {

                    DomainEvent domainEvent = mapper.readValue( value, DomainEvent.class );
                    log.debug( "process : domainEvent=" + domainEvent );

                    return new KeyValue<>( domainEvent.getBoardUuid().toString(), domainEvent );

                } catch( IOException e ) {
                    log.error( "process : error converting json to DomainEvent", e );
                }

                return null;
            })
            .groupBy( (s, domainEvent) -> s, Serialized.with( Serdes.String(), domainEventSerde ) )
            .aggregate(
                    Board::new,
                    (key, domainEvent, board) -> board.handleEvent( domainEvent ),
                    Materialized.<String, Board, KeyValueStore<Bytes, byte[]>>as( BOARD_EVENTS_SNAPSHOTS )
                        .withKeySerde( Serdes.String() )
                        .withValueSerde( boardSerde )
            );

    log.debug( "process : exit" );
}
 
Example #30
Source File: KafkaPartitionLevelConsumer.java    From incubator-pinot with Apache License 2.0 5 votes vote down vote up
private Iterable<ConsumerRecord<String, Bytes>> buildOffsetFilteringIterable(
    final List<ConsumerRecord<String, Bytes>> messageAndOffsets, final long startOffset, final long endOffset) {
  return Iterables.filter(messageAndOffsets, input -> {
    // Filter messages that are either null or have an offset ∉ [startOffset, endOffset]
    return input != null && input.offset() >= startOffset && (endOffset > input.offset() || endOffset == -1);
  });
}