Java Code Examples for org.apache.kafka.streams.KafkaStreams#store()

public AbstractKafkaRepository(String name, Serde<V> valueSerde, Function<V,String> keyFunction, KafkaAdminService kafkaAdminService, KafkaConfiguration kafkaConfiguration) {
    this.repositoryName = name + "-db";
    this.keyFunction = keyFunction;
    this.producer = KafkaUtils.kafkaProducer(kafkaConfiguration.getBootstrapServers(), StringSerializer.class, JsonNodeSerialializer.class);
    kafkaAdminService.createTopic(kafkaAdminService.buildTopicInfo(repositoryName,TopicConfig.CLEANUP_POLICY_COMPACT));

    Properties props = KafkaUtils.createKStreamProperties(repositoryName + "-stream"+ UUID.randomUUID().toString(), kafkaConfiguration.getBootstrapServers());
    StreamsBuilder builder = new StreamsBuilder();

    final GlobalKTable<String, V> globalKTable = builder.globalTable(repositoryName, materialize(valueSerde));

    final KafkaStreams streams = new KafkaStreams(builder.build(), props);
    streams.start();
    producer.flush();
    keyValueStore = streams.store(getStoreName(), QueryableStoreTypes.keyValueStore());

    Runtime.getRuntime().addShutdownHook(new Thread(streams::close));

}
 

/**
 * Query local state store to extract metrics
 *
 * @return local Metrics
 */
private Metrics getLocalMetrics() {
    HostInfo thisInstance = GlobalAppState.getInstance().getHostPortInfo();
    KafkaStreams ks = GlobalAppState.getInstance().getKafkaStreams();

    String source = thisInstance.host() + ":" + thisInstance.port();
    Metrics localMetrics = new Metrics();

    ReadOnlyKeyValueStore<String, Double> averageStore = ks
            .store(storeName,
                    QueryableStoreTypes.<String, Double>keyValueStore());

    LOGGER.log(Level.INFO, "Entries in store {0}", averageStore.approximateNumEntries());
    KeyValueIterator<String, Double> storeIterator = averageStore.all();

    while (storeIterator.hasNext()) {
        KeyValue<String, Double> kv = storeIterator.next();
        localMetrics.add(source, kv.key, String.valueOf(kv.value));

    }
    LOGGER.log(Level.INFO, "Local store state {0}", localMetrics);
    return localMetrics;
}
 

/**
 * get Metrics for a machine
 * @param machine
 * @return 
 */
private Metrics getLocalMetrics(String machine) {
    LOGGER.log(Level.INFO, "Getting Metrics for machine {0}", machine);
    
    HostInfo thisInstance = GlobalAppState.getInstance().getHostPortInfo();
    KafkaStreams ks = GlobalAppState.getInstance().getKafkaStreams();

    String source = thisInstance.host() + ":" + thisInstance.port();
    Metrics localMetrics = new Metrics();

    ReadOnlyKeyValueStore<String, Double> averageStore = ks
            .store(storeName,
                    QueryableStoreTypes.<String, Double>keyValueStore());

    LOGGER.log(Level.INFO, "Entries in store {0}", averageStore.approximateNumEntries());

    localMetrics.add(source, machine, String.valueOf(averageStore.get(machine)));

    LOGGER.log(Level.INFO, "Metrics for machine {0} - {1}", new Object[]{machine, localMetrics});
    return localMetrics;
}
 

@Override
protected ExtReadOnlyKeyValueStore<K, V> localService(String storeName, KafkaStreams streams) {
    QueryableStoreType<ReadOnlyKeyValueStore<K, V>> queryableStoreType = QueryableStoreTypes.keyValueStore();
    StoreQueryParameters<ReadOnlyKeyValueStore<K, V>> sqp = StoreQueryParameters.fromNameAndType(storeName, queryableStoreType);
    ReadOnlyKeyValueStore<K, V> delegate = streams.store(sqp);
    return new ExtReadOnlyKeyValueStoreImpl<>(delegate, filterPredicate);
}
 

/**
 * Retrieve and return a queryable store by name created in the application.  If state store is not
 * present an Optional.empty() will be returned.
 *
 * @param storeName name of the queryable store
 * @param storeType type of the queryable store
 * @param <T>       generic queryable store
 * @return queryable store.
 */
public <T> Optional<T> getQueryableStore(String storeName, QueryableStoreType<T> storeType) {
    for (KafkaStreams kafkaStream : this.streams) {
        try {
            T store = kafkaStream.store(storeName, storeType);
            if (store != null) {
                return Optional.of(store);
            }
        } catch (InvalidStateStoreException ignored) {
            //pass through
        }
    }
    return Optional.empty();
}
 

public static <K, V> List<KeyValue<K, V>> listFromStore(KafkaStreams streams, String storeName) {
    final ReadOnlyKeyValueStore<K, V> store = streams.store(
        storeName, QueryableStoreTypes.keyValueStore());

    try (final KeyValueIterator<K, V> all = store.all()) {
        List<KeyValue<K, V>> result = new ArrayList<>();
        while (all.hasNext()) {
            result.add(all.next());
        }
        return result;
    }
}
 

public static <K, V> NavigableMap<K, V> mapFromStore(KafkaStreams streams, String storeName) {
    final ReadOnlyKeyValueStore<K, V> store = streams.store(
        storeName, QueryableStoreTypes.keyValueStore());

    try (final KeyValueIterator<K, V> all = store.all()) {
        NavigableMap<K, V> result = new TreeMap<>();
        while (all.hasNext()) {
            KeyValue<K, V> next = all.next();
            result.put(next.key, next.value);
        }
        return result;
    }
}