Java Code Examples for org.apache.kafka.streams.Topology#AutoOffsetReset

private KTable createKTable(
    StreamsBuilder builder, final Topology.AutoOffsetReset autoOffsetReset,
    final KsqlTable ksqlTable,
    final Serde<GenericRow> genericRowSerde,
    final Serde<GenericRow> genericRowSerdeAfterRead
) {
  if (ksqlTable.isWindowed()) {
    return table(
        builder.stream(
            ksqlTable.getKsqlTopic().getKafkaTopicName(),
            Consumed.with(windowedSerde, genericRowSerde).withOffsetResetPolicy(autoOffsetReset)
        ).mapValues(windowedMapper).transformValues(new AddTimestampColumn()),
        windowedSerde,
        genericRowSerdeAfterRead
    );
  } else {
    return table(
        builder.stream(
            ksqlTable.getKsqlTopic().getKafkaTopicName(),
            Consumed.with(Serdes.String(), genericRowSerde).withOffsetResetPolicy(autoOffsetReset)
        ).mapValues(nonWindowedValueMapper).transformValues(new AddTimestampColumn()),
        Serdes.String(),
        genericRowSerdeAfterRead
    );
  }
}
 

protected Topology.AutoOffsetReset getAutoOffsetReset(String inboundName, KafkaStreamsConsumerProperties extendedConsumerProperties) {
	final KafkaConsumerProperties.StartOffset startOffset = extendedConsumerProperties
			.getStartOffset();
	Topology.AutoOffsetReset autoOffsetReset = null;
	if (startOffset != null) {
		switch (startOffset) {
			case earliest:
				autoOffsetReset = Topology.AutoOffsetReset.EARLIEST;
				break;
			case latest:
				autoOffsetReset = Topology.AutoOffsetReset.LATEST;
				break;
			default:
				break;
		}
	}
	if (extendedConsumerProperties.isResetOffsets()) {
		AbstractKafkaStreamsBinderProcessor.LOG.warn("Detected resetOffsets configured on binding "
				+ inboundName + ". "
				+ "Setting resetOffsets in Kafka Streams binder does not have any effect.");
	}
	return autoOffsetReset;
}
 

private KStream<?, ?> getkStream(String inboundName,
								KafkaStreamsStateStoreProperties storeSpec,
								BindingProperties bindingProperties,
								KafkaStreamsConsumerProperties kafkaStreamsConsumerProperties, StreamsBuilder streamsBuilder,
								Serde<?> keySerde, Serde<?> valueSerde,
								Topology.AutoOffsetReset autoOffsetReset, boolean firstBuild) {
	if (storeSpec != null) {
		StoreBuilder storeBuilder = buildStateStore(storeSpec);
		streamsBuilder.addStateStore(storeBuilder);
		if (LOG.isInfoEnabled()) {
			LOG.info("state store " + storeBuilder.name() + " added to topology");
		}
	}
	return getKStream(inboundName, bindingProperties, kafkaStreamsConsumerProperties, streamsBuilder,
			keySerde, valueSerde, autoOffsetReset, firstBuild);
}
 

private Topology.AutoOffsetReset getAutoOffsetReset(Map<String, Object> props) {
  if (props.containsKey(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG)) {
    final String offestReset = props.get(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG).toString();
    if (offestReset.equalsIgnoreCase("EARLIEST")) {
      return Topology.AutoOffsetReset.EARLIEST;
    } else if (offestReset.equalsIgnoreCase("LATEST")) {
      return Topology.AutoOffsetReset.LATEST;
    }
  }
  return null;
}
 

protected KStream<?, ?> getKStream(String inboundName, BindingProperties bindingProperties, KafkaStreamsConsumerProperties kafkaStreamsConsumerProperties,
		StreamsBuilder streamsBuilder, Serde<?> keySerde, Serde<?> valueSerde, Topology.AutoOffsetReset autoOffsetReset, boolean firstBuild) {
	if (firstBuild) {
		addStateStoreBeans(streamsBuilder);
	}

	KStream<?, ?> stream;
	if (this.kafkaStreamsExtendedBindingProperties
			.getExtendedConsumerProperties(inboundName).isDestinationIsPattern()) {
		final Pattern pattern = Pattern.compile(this.bindingServiceProperties.getBindingDestination(inboundName));
		stream = streamsBuilder.stream(pattern);
	}
	else {
		String[] bindingTargets = StringUtils.commaDelimitedListToStringArray(
				this.bindingServiceProperties.getBindingDestination(inboundName));
		final Consumed<?, ?> consumed = getConsumed(kafkaStreamsConsumerProperties, keySerde, valueSerde, autoOffsetReset);
		stream = streamsBuilder.stream(Arrays.asList(bindingTargets),
				consumed);
	}
	final boolean nativeDecoding = this.bindingServiceProperties
			.getConsumerProperties(inboundName).isUseNativeDecoding();
	if (nativeDecoding) {
		LOG.info("Native decoding is enabled for " + inboundName
				+ ". Inbound deserialization done at the broker.");
	}
	else {
		LOG.info("Native decoding is disabled for " + inboundName
				+ ". Inbound message conversion done by Spring Cloud Stream.");
	}

	return getkStream(bindingProperties, stream, nativeDecoding);
}
 

private <K, V> KTable<K, V> materializedAs(StreamsBuilder streamsBuilder, String destination, String storeName,
		Serde<K> k, Serde<V> v, Topology.AutoOffsetReset autoOffsetReset, KafkaStreamsConsumerProperties kafkaStreamsConsumerProperties) {

	final Consumed<K, V> consumed = getConsumed(kafkaStreamsConsumerProperties, k, v, autoOffsetReset);
	return streamsBuilder.table(this.bindingServiceProperties.getBindingDestination(destination),
			consumed, getMaterialized(storeName, k, v));
}
 

private <K, V> GlobalKTable<K, V> materializedAsGlobalKTable(
		StreamsBuilder streamsBuilder, String destination, String storeName,
		Serde<K> k, Serde<V> v, Topology.AutoOffsetReset autoOffsetReset, KafkaStreamsConsumerProperties kafkaStreamsConsumerProperties) {
	final Consumed<K, V> consumed = getConsumed(kafkaStreamsConsumerProperties, k, v, autoOffsetReset);
	return streamsBuilder.globalTable(
			this.bindingServiceProperties.getBindingDestination(destination),
			consumed,
			getMaterialized(storeName, k, v));
}
 

private GlobalKTable<?, ?> getGlobalKTable(KafkaStreamsConsumerProperties kafkaStreamsConsumerProperties,
		StreamsBuilder streamsBuilder,
		Serde<?> keySerde, Serde<?> valueSerde, String materializedAs,
		String bindingDestination, Topology.AutoOffsetReset autoOffsetReset) {
	final Consumed<?, ?> consumed = getConsumed(kafkaStreamsConsumerProperties, keySerde, valueSerde, autoOffsetReset);
	return materializedAs != null
			? materializedAsGlobalKTable(streamsBuilder, bindingDestination,
			materializedAs, keySerde, valueSerde, autoOffsetReset, kafkaStreamsConsumerProperties)
			: streamsBuilder.globalTable(bindingDestination,
			consumed);
}
 

private KTable<?, ?> getKTable(KafkaStreamsConsumerProperties kafkaStreamsConsumerProperties,
		StreamsBuilder streamsBuilder, Serde<?> keySerde,
		Serde<?> valueSerde, String materializedAs, String bindingDestination,
		Topology.AutoOffsetReset autoOffsetReset) {
	final Consumed<?, ?> consumed = getConsumed(kafkaStreamsConsumerProperties, keySerde, valueSerde, autoOffsetReset);
	return materializedAs != null
			? materializedAs(streamsBuilder, bindingDestination, materializedAs,
			keySerde, valueSerde, autoOffsetReset, kafkaStreamsConsumerProperties)
			: streamsBuilder.table(bindingDestination,
			consumed);
}
 

private <K, V> Consumed<K, V> getConsumed(KafkaStreamsConsumerProperties kafkaStreamsConsumerProperties,
		Serde<K> keySerde, Serde<V> valueSerde, Topology.AutoOffsetReset autoOffsetReset) {
	TimestampExtractor timestampExtractor = null;
	if (!StringUtils.isEmpty(kafkaStreamsConsumerProperties.getTimestampExtractorBeanName())) {
		timestampExtractor = applicationContext.getBean(kafkaStreamsConsumerProperties.getTimestampExtractorBeanName(),
				TimestampExtractor.class);
	}
	final Consumed<K, V> consumed = Consumed.with(keySerde, valueSerde)
			.withOffsetResetPolicy(autoOffsetReset);
	if (timestampExtractor != null) {
		consumed.withTimestampExtractor(timestampExtractor);
	}
	return consumed;
}