Java Code Examples for io.confluent.kafka.streams.serdes.avro.SpecificAvroSerde#configure()

The following examples show how to use io.confluent.kafka.streams.serdes.avro.SpecificAvroSerde#configure() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: AggregatingCountTest.java    From kafka-tutorials with Apache License 2.0 7 votes vote down vote up 
private SpecificAvroSerde<TicketSale> makeSerializer(Properties envProps)
    throws IOException, RestClientException {

  final MockSchemaRegistryClient client = new MockSchemaRegistryClient();
  String inputTopic = envProps.getProperty("input.topic.name");
  String outputTopic = envProps.getProperty("output.topic.name");

  final Schema schema = TicketSale.SCHEMA$;
  client.register(inputTopic + "-value", schema);
  client.register(outputTopic + "-value", schema);

  SpecificAvroSerde<TicketSale> serde = new SpecificAvroSerde<>(client);

  Map<String, String> config = new HashMap<>();
  config.put("schema.registry.url", envProps.getProperty("schema.registry.url"));
  serde.configure(config, false);

  return serde;
}
Example 2
Source File: AggregatingSumTest.java    From kafka-tutorials with Apache License 2.0 7 votes vote down vote up 
private SpecificAvroSerde<TicketSale> makeSerializer(Properties envProps)
    throws IOException, RestClientException {

  final MockSchemaRegistryClient client = new MockSchemaRegistryClient();
  String inputTopic = envProps.getProperty("input.topic.name");
  String outputTopic = envProps.getProperty("output.topic.name");

  final Schema schema = TicketSale.SCHEMA$;
  client.register(inputTopic + "-value", schema);
  client.register(outputTopic + "-value", schema);

  SpecificAvroSerde<TicketSale> serde = new SpecificAvroSerde<>(client);

  Map<String, String> config = new HashMap<>();
  config.put("schema.registry.url", envProps.getProperty("schema.registry.url"));
  serde.configure(config, false);

  return serde;
}
Example 3
Source File: StreamsIngestTest.java    From kafka-tutorials with Apache License 2.0 6 votes vote down vote up 
private SpecificAvroSerde<City> makeSerializer(Properties envProps)
    throws IOException, RestClientException {

  final MockSchemaRegistryClient client = new MockSchemaRegistryClient();
  String inputTopic = envProps.getProperty("input.topic.name");
  String outputTopic = envProps.getProperty("output.topic.name");

  final Schema schema = City.SCHEMA$;
  client.register(inputTopic + "-value", schema);
  client.register(outputTopic + "-value", schema);

  SpecificAvroSerde<City> serde = new SpecificAvroSerde<>(client);

  Map<String, String> config = new HashMap<>();
  config.put("schema.registry.url", envProps.getProperty("schema.registry.url"));
  serde.configure(config, false);

  return serde;
}
Example 4
Source File: FkJoinTableToTable.java    From kafka-tutorials with Apache License 2.0 5 votes vote down vote up 
static <T extends SpecificRecord> SpecificAvroSerde<T> getSpecificAvroSerde(final Properties envProps) {
    final SpecificAvroSerde<T> specificAvroSerde = new SpecificAvroSerde<>();

    final HashMap<String, String> serdeConfig = new HashMap<>();
    serdeConfig.put(AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG,
            envProps.getProperty("schema.registry.url"));

    specificAvroSerde.configure(serdeConfig, false);
    return specificAvroSerde;
}
Example 5
Source File: DynamicOutputTopic.java    From kafka-tutorials with Apache License 2.0 5 votes vote down vote up 
static <T extends SpecificRecord> SpecificAvroSerde<T> getSpecificAvroSerde(final Properties envProps) {
    final SpecificAvroSerde<T> specificAvroSerde = new SpecificAvroSerde<>();

    final HashMap<String, String> serdeConfig = new HashMap<>();
    serdeConfig.put(AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG,
            envProps.getProperty("schema.registry.url"));

    specificAvroSerde.configure(serdeConfig, false);
    return specificAvroSerde;
}
Example 6
Source File: CogroupingStreams.java    From kafka-tutorials with Apache License 2.0 5 votes vote down vote up 
static <T extends SpecificRecord> SpecificAvroSerde<T> getSpecificAvroSerde(final Properties envProps) {
    final SpecificAvroSerde<T> specificAvroSerde = new SpecificAvroSerde<>();

    final HashMap<String, String> serdeConfig = new HashMap<>();
    serdeConfig.put(AbstractKafkaSchemaSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG,
            envProps.getProperty("schema.registry.url"));

    specificAvroSerde.configure(serdeConfig, false);
    return specificAvroSerde;
}
Example 7
Source File: StreamsIngest.java    From kafka-tutorials with Apache License 2.0 5 votes vote down vote up 
private SpecificAvroSerde<City> citySerde(final Properties envProps) {
  final SpecificAvroSerde<City> serde = new SpecificAvroSerde<>();
  Map<String, String> config = new HashMap<>();
  config.put(AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, envProps.getProperty("schema.registry.url"));
  serde.configure(config, false);
  return serde;
}
Example 8
Source File: FindDistinctEvents.java    From kafka-tutorials with Apache License 2.0 5 votes vote down vote up 
private SpecificAvroSerde<Click> buildClicksSerde(final Properties envProps) {
    final SpecificAvroSerde<Click> serde = new SpecificAvroSerde<>();
    Map<String, String> config = new HashMap<>();
    config.put(AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, envProps.getProperty("schema.registry.url"));
    serde.configure(config, false);
    return serde;
}
Example 9
Source File: AggregatingSum.java    From kafka-tutorials with Apache License 2.0 5 votes vote down vote up 
private SpecificAvroSerde<TicketSale> ticketSaleSerde(final Properties envProps) {
  final SpecificAvroSerde<TicketSale> serde = new SpecificAvroSerde<>();
  Map<String, String> config = new HashMap<>();
  config.put(AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, envProps.getProperty("schema.registry.url"));
  serde.configure(config, false);
  return serde;
}
Example 10
Source File: TumblingWindow.java    From kafka-tutorials with Apache License 2.0 5 votes vote down vote up 
private SpecificAvroSerde<Rating> ratedMovieAvroSerde(Properties envProps) {
    SpecificAvroSerde<Rating> movieAvroSerde = new SpecificAvroSerde<>();

    final HashMap<String, String> serdeConfig = new HashMap<>();
    serdeConfig.put(AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG,
            envProps.getProperty("schema.registry.url"));

    movieAvroSerde.configure(serdeConfig, false);
    return movieAvroSerde;
}
Example 11
Source File: CountVersionApplication.java    From spring-cloud-stream-samples with Apache License 2.0 5 votes vote down vote up 
@Bean
public Function<KStream<Object, Sensor>, KStream<String, Long>> process() {

	//The following Serde definitions are not needed in the topoloyy below
	//as we are not using it. However, if your topoloyg explicitly uses this
	//Serde, you need to configure this with the schema registry url as below.

	final Map<String, String> serdeConfig = Collections.singletonMap(
			AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, "http://localhost:8081");

	final SpecificAvroSerde<Sensor> sensorSerde = new SpecificAvroSerde<>();
	sensorSerde.configure(serdeConfig, false);

	return input -> input
			.map((k, value) -> {
				String newKey = "v1";
				if (value.getId().toString().endsWith("v2")) {
					newKey = "v2";
				}
				return new KeyValue<>(newKey, value);
			})
			.groupByKey()
			.count(Materialized.<String, Long, KeyValueStore<Bytes, byte[]>>as(STORE_NAME)
					.withKeySerde(Serdes.String())
					.withValueSerde(Serdes.Long()))
			.toStream();

}
Example 12
Source File: AggregatingMinMaxTest.java    From kafka-tutorials with Apache License 2.0 5 votes vote down vote up 
private static SpecificAvroSerde<YearlyMovieFigures> makeYearlyMovieFiguresSerializer(
        Properties envProps, SchemaRegistryClient srClient) {

  SpecificAvroSerde<YearlyMovieFigures> serde = new SpecificAvroSerde<>(srClient);
  serde.configure(Collections.singletonMap(
          "schema.registry.url", envProps.getProperty("schema.registry.url")), false);
  return serde;
}
Example 13
Source File: AggregatingMinMaxTest.java    From kafka-tutorials with Apache License 2.0 5 votes vote down vote up 
private static SpecificAvroSerde<MovieTicketSales> makeMovieTicketSalesSerializer(
        Properties envProps, SchemaRegistryClient srClient) {

  SpecificAvroSerde<MovieTicketSales> serde = new SpecificAvroSerde<>(srClient);
  serde.configure(Collections.singletonMap(
          "schema.registry.url", envProps.getProperty("schema.registry.url")), false);
  return serde;
}
Example 14
Source File: AggregatingMinMax.java    From kafka-tutorials with Apache License 2.0 5 votes vote down vote up 
public static SpecificAvroSerde<YearlyMovieFigures> movieFiguresSerde(final Properties envProps) {
  final SpecificAvroSerde<YearlyMovieFigures> serde = new SpecificAvroSerde<>();
  serde.configure(Collections.singletonMap(
    AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, envProps.getProperty("schema.registry.url")
  ), false);
  return serde;
}
Example 15
Source File: AggregatingMinMax.java    From kafka-tutorials with Apache License 2.0 5 votes vote down vote up 
public static SpecificAvroSerde<MovieTicketSales> ticketSaleSerde(final Properties envProps) {
  final SpecificAvroSerde<MovieTicketSales> serde = new SpecificAvroSerde<>();
  serde.configure(Collections.singletonMap(
          AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG,
          envProps.getProperty("schema.registry.url")), false);
  return serde;
}
Example 16
Source File: FilterEvents.java    From kafka-tutorials with Apache License 2.0 5 votes vote down vote up 
private SpecificAvroSerde<Publication> publicationSerde(final Properties envProps) {
  final SpecificAvroSerde<Publication> serde = new SpecificAvroSerde<>();
  Map<String, String> config = new HashMap<>();
  config.put(AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, envProps.getProperty("schema.registry.url"));
  serde.configure(config, false);
  return serde;
}
Example 17
Source File: RunningAverage.java    From kafka-tutorials with Apache License 2.0 4 votes vote down vote up 
public static SpecificAvroSerde<CountAndSum> getCountAndSumSerde(Properties envProps) {
  SpecificAvroSerde<CountAndSum> serde = new SpecificAvroSerde<>();
  serde.configure(getSerdeConfig(envProps), false);
  return serde;
}
Example 18
Source File: RunningAverage.java    From kafka-tutorials with Apache License 2.0 4 votes vote down vote up 
public static SpecificAvroSerde<Rating> getRatingSerde(Properties envProps) {
  SpecificAvroSerde<Rating> serde = new SpecificAvroSerde<>();
  serde.configure(getSerdeConfig(envProps), false);
  return serde;
}
Example 19
Source File: KafkaStreamsInteractiveQuerySample.java    From spring-cloud-stream-samples with Apache License 2.0 4 votes vote down vote up 
@Bean
public BiConsumer<KStream<String, PlayEvent>, KTable<Long, Song>> process() {

	return (s, t) -> {
		// create and configure the SpecificAvroSerdes required in this example
		final Map<String, String> serdeConfig = Collections.singletonMap(
				AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, "http://localhost:8081");

		final SpecificAvroSerde<PlayEvent> playEventSerde = new SpecificAvroSerde<>();
		playEventSerde.configure(serdeConfig, false);

		final SpecificAvroSerde<Song> keySongSerde = new SpecificAvroSerde<>();
		keySongSerde.configure(serdeConfig, true);

		final SpecificAvroSerde<Song> valueSongSerde = new SpecificAvroSerde<>();
		valueSongSerde.configure(serdeConfig, false);

		final SpecificAvroSerde<SongPlayCount> songPlayCountSerde = new SpecificAvroSerde<>();
		songPlayCountSerde.configure(serdeConfig, false);

		// Accept play events that have a duration >= the minimum
		final KStream<Long, PlayEvent> playsBySongId =
				s.filter((region, event) -> event.getDuration() >= MIN_CHARTABLE_DURATION)
						// repartition based on song id
						.map((key, value) -> KeyValue.pair(value.getSongId(), value));

		// join the plays with song as we will use it later for charting
		final KStream<Long, Song> songPlays = playsBySongId.leftJoin(t,
				(value1, song) -> song,
				Joined.with(Serdes.Long(), playEventSerde, valueSongSerde));

		// create a state store to track song play counts
		final KTable<Song, Long> songPlayCounts = songPlays.groupBy((songId, song) -> song,
				Serialized.with(keySongSerde, valueSongSerde))
				.count(Materialized.<Song, Long, KeyValueStore<Bytes, byte[]>>as(SONG_PLAY_COUNT_STORE)
						.withKeySerde(valueSongSerde)
						.withValueSerde(Serdes.Long()));

		final TopFiveSerde topFiveSerde = new TopFiveSerde();

		// Compute the top five charts for each genre. The results of this computation will continuously update the state
		// store "top-five-songs-by-genre", and this state store can then be queried interactively via a REST API (cf.
		// MusicPlaysRestService) for the latest charts per genre.
		songPlayCounts.groupBy((song, plays) ->
						KeyValue.pair(song.getGenre().toLowerCase(),
								new SongPlayCount(song.getId(), plays)),
				Serialized.with(Serdes.String(), songPlayCountSerde))
				// aggregate into a TopFiveSongs instance that will keep track
				// of the current top five for each genre. The data will be available in the
				// top-five-songs-genre store
				.aggregate(TopFiveSongs::new,
						(aggKey, value, aggregate) -> {
							aggregate.add(value);
							return aggregate;
						},
						(aggKey, value, aggregate) -> {
							aggregate.remove(value);
							return aggregate;
						},
						Materialized.<String, TopFiveSongs, KeyValueStore<Bytes, byte[]>>as(TOP_FIVE_SONGS_BY_GENRE_STORE)
								.withKeySerde(Serdes.String())
								.withValueSerde(topFiveSerde)
				);

		// Compute the top five chart. The results of this computation will continuously update the state
		// store "top-five-songs", and this state store can then be queried interactively via a REST API (cf.
		// MusicPlaysRestService) for the latest charts per genre.
		songPlayCounts.groupBy((song, plays) ->
						KeyValue.pair(TOP_FIVE_KEY,
								new SongPlayCount(song.getId(), plays)),
				Serialized.with(Serdes.String(), songPlayCountSerde))
				.aggregate(TopFiveSongs::new,
						(aggKey, value, aggregate) -> {
							aggregate.add(value);
							return aggregate;
						},
						(aggKey, value, aggregate) -> {
							aggregate.remove(value);
							return aggregate;
						},
						Materialized.<String, TopFiveSongs, KeyValueStore<Bytes, byte[]>>as(TOP_FIVE_SONGS_STORE)
								.withKeySerde(Serdes.String())
								.withValueSerde(topFiveSerde)
				);
	};

}
Example 20
Source File: WindowFinalResult.java    From kafka-tutorials with Apache License 2.0 3 votes vote down vote up 
public static void main(String[] args) {

        final Config config = ConfigFactory.load();

        final Properties properties = buildProperties(config);

        Map<String, Object> serdeConfig =
                singletonMap(SCHEMA_REGISTRY_URL_CONFIG, config.getString("schema.registry.url"));

        SpecificAvroSerde<PressureAlert> pressureSerde = new SpecificAvroSerde<>();

        pressureSerde.configure(serdeConfig, false);

        TimeWindows windows = TimeWindows

                .of(config.getDuration("window.size"))

                .advanceBy(config.getDuration("window.size"))

                .grace(config.getDuration("window.grace.period"));

        Topology topology = buildTopology(config, windows, pressureSerde);

        logger.debug(topology.describe().toString());

        final KafkaStreams streams = new KafkaStreams(topology, properties);

        Runtime.getRuntime().addShutdownHook(new Thread(streams::close));

        streams.cleanUp();
        streams.start();
    }