org.apache.kafka.common.serialization.DoubleSerializer Java Examples
The following examples show how to use
org.apache.kafka.common.serialization.DoubleSerializer.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
| Source File: KafkaProducerInterceptorWrapper.java From pulsar with Apache License 2.0 | 6 votes |
|
static Deserializer getDeserializer(Serializer serializer) {
if (serializer instanceof StringSerializer) {
return new StringDeserializer();
} else if (serializer instanceof LongSerializer) {
return new LongDeserializer();
} else if (serializer instanceof IntegerSerializer) {
return new IntegerDeserializer();
} else if (serializer instanceof DoubleSerializer) {
return new DoubleDeserializer();
} else if (serializer instanceof BytesSerializer) {
return new BytesDeserializer();
} else if (serializer instanceof ByteBufferSerializer) {
return new ByteBufferDeserializer();
} else if (serializer instanceof ByteArraySerializer) {
return new ByteArrayDeserializer();
} else {
throw new IllegalArgumentException(serializer.getClass().getName() + " is not a valid or supported subclass of org.apache.kafka.common.serialization.Serializer.");
}
}
Example #2
| Source File: KafkaProducerInterceptorWrapperTest.java From pulsar with Apache License 2.0 | 6 votes |
|
@DataProvider(name = "serializers")
public Object[][] serializers() {
return new Object[][] {
{
new StringSerializer(), StringDeserializer.class
},
{
new LongSerializer(), LongDeserializer.class
},
{
new IntegerSerializer(), IntegerDeserializer.class,
},
{
new DoubleSerializer(), DoubleDeserializer.class,
},
{
new BytesSerializer(), BytesDeserializer.class
},
{
new ByteBufferSerializer(), ByteBufferDeserializer.class
},
{
new ByteArraySerializer(), ByteArrayDeserializer.class
}
};
}
Example #3
| Source File: KafkaUsage.java From smallrye-reactive-messaging with Apache License 2.0 | 5 votes |
|
public void produceDoubles(int messageCount, Runnable completionCallback,
Supplier<ProducerRecord<String, Double>> messageSupplier) {
Serializer<String> keySer = new StringSerializer();
Serializer<Double> valSer = new DoubleSerializer();
String randomId = UUID.randomUUID().toString();
this.produce(randomId, messageCount, keySer, valSer, completionCallback, messageSupplier);
}
Example #4
| Source File: PageRankTest.java From kafka-graphs with Apache License 2.0 | 4 votes |
|
@Test
public void testChainPageRank() throws Exception {
String suffix = "chain";
StreamsBuilder builder = new StreamsBuilder();
Properties producerConfig = ClientUtils.producerConfig(CLUSTER.bootstrapServers(), LongSerializer.class,
DoubleSerializer.class, new Properties()
);
KTable<Edge<Long>, Double> edges =
StreamUtils.tableFromCollection(builder, producerConfig, new KryoSerde<>(), Serdes.Double(),
TestGraphUtils.getChain());
KGraph<Long, Double, Double> initialGraph = KGraph.fromEdges(edges,
GraphAlgorithmType.initialVertexValueMapper(GraphAlgorithmType.pagerank),
GraphSerialized.with(Serdes.Long(), Serdes.Double(), Serdes.Double()));
KTable<Long, Tuple2<Double, Double>> vertices =
initialGraph.vertices().mapValues((k, v) -> new Tuple2<>(0.0, 0.0));
KGraph<Long, Tuple2<Double, Double>, Double> graph =
new KGraph<>(vertices, initialGraph.edges(), GraphSerialized.with(initialGraph.keySerde(), new KryoSerde<>
(), Serdes.Double()));
Properties props = ClientUtils.streamsConfig("prepare-" + suffix, "prepare-client-" + suffix, CLUSTER
.bootstrapServers(),
graph.keySerde().getClass(), graph.vertexValueSerde().getClass());
CompletableFuture<Map<TopicPartition, Long>> state = GraphUtils.groupEdgesBySourceAndRepartition(builder, props, graph, "vertices-" + suffix, "edgesGroupedBySource-" + suffix, 50, (short) 1);
Map<TopicPartition, Long> offsets = state.get();
double resetProb = 0.15;
double tol = 0.0001;
Map<String, Object> configs = new HashMap<>();
configs.put(PageRank.RESET_PROBABILITY, resetProb);
configs.put(PageRank.TOLERANCE, tol);
Optional<Double> initMsg = Optional.of(resetProb / (1.0 - resetProb));
algorithm =
new PregelGraphAlgorithm<>(null, "run-" + suffix, CLUSTER.bootstrapServers(),
CLUSTER.zKConnectString(), "vertices-" + suffix, "edgesGroupedBySource-" + suffix, offsets, graph.serialized(),
"solutionSet-" + suffix, "solutionSetStore-" + suffix, "workSet-" + suffix, 50, (short) 1,
configs, initMsg, new PageRank<>());
props = ClientUtils.streamsConfig("run-" + suffix, "run-client-" + suffix, CLUSTER.bootstrapServers(),
graph.keySerde().getClass(), KryoSerde.class);
KafkaStreams streams = algorithm.configure(new StreamsBuilder(), props).streams();
int maxIterations = 2; //Integer.MAX_VALUE;
GraphAlgorithmState<KTable<Long, Tuple2<Double, Double>>> ranks = algorithm.run(maxIterations);
ranks.result().get();
Thread.sleep(2000);
Map<Long, Tuple2<Double, Double>> map = StreamUtils.mapFromStore(ranks.streams(), "solutionSetStore-" +
suffix);
List<Double> list = map.values().stream().map(Tuple2::_1).sorted().collect(Collectors.toList());
log.debug("result: {}", map);
List<Double> expectedResult = new ArrayList<>();
expectedResult.add(0.15);
expectedResult.add(0.27749999999999997);
expectedResult.add(0.27749999999999997);
expectedResult.add(0.27749999999999997);
expectedResult.add(0.27749999999999997);
expectedResult.add(0.27749999999999997);
expectedResult.add(0.27749999999999997);
expectedResult.add(0.27749999999999997);
expectedResult.add(0.27749999999999997);
expectedResult.add(0.27749999999999997);
assertEquals(expectedResult, list);
}
Example #5
| Source File: PageRankTest.java From kafka-graphs with Apache License 2.0 | 4 votes |
|
@Test
public void testChainLongerPageRank() throws Exception {
String suffix = "longer";
StreamsBuilder builder = new StreamsBuilder();
Properties producerConfig = ClientUtils.producerConfig(CLUSTER.bootstrapServers(), LongSerializer.class,
DoubleSerializer.class, new Properties()
);
KTable<Edge<Long>, Double> edges =
StreamUtils.tableFromCollection(builder, producerConfig, new KryoSerde<>(), Serdes.Double(),
TestGraphUtils.getChain());
KGraph<Long, Double, Double> initialGraph = KGraph.fromEdges(edges,
GraphAlgorithmType.initialVertexValueMapper(GraphAlgorithmType.pagerank),
GraphSerialized.with(Serdes.Long(), Serdes.Double(), Serdes.Double()));
KTable<Long, Tuple2<Double, Double>> vertices =
initialGraph.vertices().mapValues((k, v) -> new Tuple2<>(0.0, 0.0));
KGraph<Long, Tuple2<Double, Double>, Double> graph =
new KGraph<>(vertices, initialGraph.edges(), GraphSerialized.with(initialGraph.keySerde(), new KryoSerde<>
(), Serdes.Double()));
Properties props = ClientUtils.streamsConfig("prepare-" + suffix, "prepare-client-" + suffix, CLUSTER
.bootstrapServers(),
graph.keySerde().getClass(), graph.vertexValueSerde().getClass());
CompletableFuture<Map<TopicPartition, Long>> state = GraphUtils.groupEdgesBySourceAndRepartition(builder, props, graph, "vertices-" + suffix, "edgesGroupedBySource-" + suffix, 50, (short) 1);
Map<TopicPartition, Long> offsets = state.get();
double resetProb = 0.15;
double tol = 0.0001;
Map<String, Object> configs = new HashMap<>();
configs.put(PageRank.RESET_PROBABILITY, resetProb);
configs.put(PageRank.TOLERANCE, tol);
Optional<Double> initMsg = Optional.of(resetProb / (1.0 - resetProb));
algorithm =
new PregelGraphAlgorithm<>(null, "run-" + suffix, CLUSTER.bootstrapServers(),
CLUSTER.zKConnectString(), "vertices-" + suffix, "edgesGroupedBySource-" + suffix, offsets, graph.serialized(),
"solutionSet-" + suffix, "solutionSetStore-" + suffix, "workSet-" + suffix, 50, (short) 1,
configs, initMsg, new PageRank<>());
props = ClientUtils.streamsConfig("run-" + suffix, "run-client-" + suffix, CLUSTER.bootstrapServers(),
graph.keySerde().getClass(), KryoSerde.class);
KafkaStreams streams = algorithm.configure(new StreamsBuilder(), props).streams();
int maxIterations = 11;
GraphAlgorithmState<KTable<Long, Tuple2<Double, Double>>> ranks = algorithm.run(maxIterations);
ranks.result().get();
Thread.sleep(2000);
Map<Long, Tuple2<Double, Double>> map = StreamUtils.mapFromStore(ranks.streams(), "solutionSetStore-" +
suffix);
List<Double> list = map.values().stream().map(Tuple2::_1).sorted().collect(Collectors.toList());
log.debug("result: {}", map);
List<Double> expectedResult = new ArrayList<>();
expectedResult.add(0.15);
expectedResult.add(0.27749999999999997);
expectedResult.add(0.38587499999999997);
expectedResult.add(0.47799375);
expectedResult.add(0.5562946875);
expectedResult.add(0.622850484375);
expectedResult.add(0.67942291171875);
expectedResult.add(0.7275094749609375);
expectedResult.add(0.7683830537167969);
expectedResult.add(0.8031255956592774);
assertEquals(expectedResult, list);
}
Example #6
| Source File: PageRankTest.java From kafka-graphs with Apache License 2.0 | 4 votes |
|
@Test
public void testChainPersonalPageRank() throws Exception {
String suffix = "chain-personal";
StreamsBuilder builder = new StreamsBuilder();
Properties producerConfig = ClientUtils.producerConfig(CLUSTER.bootstrapServers(), LongSerializer.class,
DoubleSerializer.class, new Properties()
);
KTable<Edge<Long>, Double> edges =
StreamUtils.tableFromCollection(builder, producerConfig, new KryoSerde<>(), Serdes.Double(),
TestGraphUtils.getChain());
KGraph<Long, Double, Double> initialGraph = KGraph.fromEdges(edges,
GraphAlgorithmType.initialVertexValueMapper(GraphAlgorithmType.pagerank),
GraphSerialized.with(Serdes.Long(), Serdes.Double(), Serdes.Double()));
long srcVertexId = 4L;
KTable<Long, Tuple2<Double, Double>> vertices =
initialGraph.vertices().mapValues((k, v) -> new Tuple2<>(0.0, k == srcVertexId ? Double.NEGATIVE_INFINITY
: 0.0));
KGraph<Long, Tuple2<Double, Double>, Double> graph =
new KGraph<>(vertices, initialGraph.edges(), GraphSerialized.with(initialGraph.keySerde(), new KryoSerde<>
(), Serdes.Double()));
Properties props = ClientUtils.streamsConfig("prepare-" + suffix, "prepare-client-" + suffix, CLUSTER
.bootstrapServers(),
graph.keySerde().getClass(), graph.vertexValueSerde().getClass());
CompletableFuture<Map<TopicPartition, Long>> state = GraphUtils.groupEdgesBySourceAndRepartition(builder, props, graph, "vertices-" + suffix, "edgesGroupedBySource-" + suffix, 50, (short) 1);
Map<TopicPartition, Long> offsets = state.get();
double resetProb = 0.15;
double tol = 0.0001;
Map<String, Object> configs = new HashMap<>();
configs.put(PageRank.RESET_PROBABILITY, resetProb);
configs.put(PageRank.TOLERANCE, tol);
configs.put(PageRank.SRC_VERTEX_ID, srcVertexId);
algorithm =
new PregelGraphAlgorithm<>(null, "run-" + suffix, CLUSTER.bootstrapServers(),
CLUSTER.zKConnectString(), "vertices-" + suffix, "edgesGroupedBySource-" + suffix, offsets, graph.serialized(),
"solutionSet-" + suffix, "solutionSetStore-" + suffix, "workSet-" + suffix, 50, (short) 1,
configs, Optional.of(0.0), new PageRank<>());
props = ClientUtils.streamsConfig("run-" + suffix, "run-client-" + suffix, CLUSTER.bootstrapServers(),
graph.keySerde().getClass(), KryoSerde.class);
KafkaStreams streams = algorithm.configure(new StreamsBuilder(), props).streams();
int maxIterations = 4;
GraphAlgorithmState<KTable<Long, Tuple2<Double, Double>>> ranks = algorithm.run(maxIterations);
ranks.result().get();
Thread.sleep(2000);
Map<Long, Tuple2<Double, Double>> map = StreamUtils.mapFromStore(ranks.streams(), "solutionSetStore-" +
suffix);
List<Double> list = map.values().stream().map(Tuple2::_1).sorted().collect(Collectors.toList());
log.debug("result: {}", map);
List<Double> expectedResult = new ArrayList<>();
expectedResult.add(0.0);
expectedResult.add(0.0);
expectedResult.add(0.0);
expectedResult.add(0.0);
expectedResult.add(0.0);
expectedResult.add(0.0);
expectedResult.add(0.6141249999999999);
expectedResult.add(0.7224999999999999);
expectedResult.add(0.85);
expectedResult.add(1.0);
assertEquals(expectedResult, list);
}
Example #7
| Source File: AdamicAdarTest.java From kafka-graphs with Apache License 2.0 | 4 votes |
|
@Test
public void testExactSimilarity() throws Exception {
String suffix = "similarity";
StreamsBuilder builder = new StreamsBuilder();
List<KeyValue<Edge<Long>, Double>> list = new ArrayList<>();
list.add(new KeyValue<>(new Edge<>(1L, 2L), 0.0));
list.add(new KeyValue<>(new Edge<>(2L, 1L), 0.0));
list.add(new KeyValue<>(new Edge<>(1L, 3L), 0.0));
list.add(new KeyValue<>(new Edge<>(3L, 1L), 0.0));
list.add(new KeyValue<>(new Edge<>(1L, 4L), 0.0));
list.add(new KeyValue<>(new Edge<>(4L, 1L), 0.0));
list.add(new KeyValue<>(new Edge<>(2L, 4L), 0.0));
list.add(new KeyValue<>(new Edge<>(4L, 2L), 0.0));
list.add(new KeyValue<>(new Edge<>(2L, 5L), 0.0));
list.add(new KeyValue<>(new Edge<>(5L, 2L), 0.0));
list.add(new KeyValue<>(new Edge<>(3L, 4L), 0.0));
list.add(new KeyValue<>(new Edge<>(4L, 3L), 0.0));
list.add(new KeyValue<>(new Edge<>(4L, 5L), 0.0));
list.add(new KeyValue<>(new Edge<>(5L, 4L), 0.0));
Properties producerConfig = ClientUtils.producerConfig(CLUSTER.bootstrapServers(), KryoSerializer.class,
DoubleSerializer.class, new Properties()
);
KTable<Edge<Long>, Double> edges =
StreamUtils.tableFromCollection(builder, producerConfig, new KryoSerde<>(), Serdes.Double(), list);
KGraph<Long, Double, Double> graph = KGraph.fromEdges(edges, new InitVertices(),
GraphSerialized.with(Serdes.Long(), Serdes.Double(), Serdes.Double()));
Properties props = ClientUtils.streamsConfig("prepare-" + suffix, "prepare-client-" + suffix,
CLUSTER.bootstrapServers(), graph.keySerde().getClass(), graph.vertexValueSerde().getClass());
CompletableFuture<Map<TopicPartition, Long>> state = GraphUtils.groupEdgesBySourceAndRepartition(builder, props, graph, "vertices-" + suffix, "edgesGroupedBySource-" + suffix, 2, (short) 1);
Map<TopicPartition, Long> offsets = state.get();
Map<String, Object> configs = new HashMap<>();
configs.put("distance.conversion.enabled", false);
algorithm =
new PregelGraphAlgorithm<>(null, "run-" + suffix, CLUSTER.bootstrapServers(),
CLUSTER.zKConnectString(), "vertices-" + suffix, "edgesGroupedBySource-" + suffix, offsets, graph.serialized(),
"solutionSet-" + suffix, "solutionSetStore-" + suffix, "workSet-" + suffix, 2, (short) 1,
configs, Optional.empty(), new AdamicAdar());
streamsConfiguration = ClientUtils.streamsConfig("run-" + suffix, "run-client-" + suffix,
CLUSTER.bootstrapServers(), graph.keySerde().getClass(), KryoSerde.class);
KafkaStreams streams = algorithm.configure(new StreamsBuilder(), streamsConfiguration).streams();
GraphAlgorithmState<KTable<Long, Double>> paths = algorithm.run();
paths.result().get();
Thread.sleep(2000);
Map<Long, Map<Long, Long>> map = StreamUtils.mapFromStore(paths.streams(), "solutionSetStore-" + suffix);
log.debug("result: {}", map);
assertEquals("{1=-1.0986122886681098, 2=-1.0986122886681098, 3=-0.6931471805599453, 4=-1.3862943611198906, 5=-0.6931471805599453}", map.toString());
Map<Long, Map<Long, Double>> edgesMap = StreamUtils.mapFromStore(paths.streams(), "edgesStore-run-" + suffix);
log.debug("edges : {}", edgesMap);
assertEquals("{1={2=-1.3862943611198906, 3=-1.3862943611198906, 4=-1.791759469228055}, 2={1=-1.3862943611198906, 4=-1.791759469228055, 5=-1.3862943611198906}, 3={1=-1.3862943611198906, 4=-1.0986122886681098}, 4={1=-1.791759469228055, 2=-1.791759469228055, 3=-1.0986122886681098, 5=-1.0986122886681098}, 5={2=-1.3862943611198906, 4=-1.0986122886681098}}", edgesMap.toString());
}
Example #8
| Source File: AdamicAdarTest.java From kafka-graphs with Apache License 2.0 | 4 votes |
|
@Test
public void testExactDistance() throws Exception {
String suffix = "distance";
StreamsBuilder builder = new StreamsBuilder();
List<KeyValue<Edge<Long>, Double>> list = new ArrayList<>();
list.add(new KeyValue<>(new Edge<>(1L, 2L), 0.0));
list.add(new KeyValue<>(new Edge<>(2L, 1L), 0.0));
list.add(new KeyValue<>(new Edge<>(1L, 3L), 0.0));
list.add(new KeyValue<>(new Edge<>(3L, 1L), 0.0));
list.add(new KeyValue<>(new Edge<>(1L, 4L), 0.0));
list.add(new KeyValue<>(new Edge<>(4L, 1L), 0.0));
list.add(new KeyValue<>(new Edge<>(2L, 4L), 0.0));
list.add(new KeyValue<>(new Edge<>(4L, 2L), 0.0));
list.add(new KeyValue<>(new Edge<>(2L, 5L), 0.0));
list.add(new KeyValue<>(new Edge<>(5L, 2L), 0.0));
list.add(new KeyValue<>(new Edge<>(3L, 4L), 0.0));
list.add(new KeyValue<>(new Edge<>(4L, 3L), 0.0));
list.add(new KeyValue<>(new Edge<>(4L, 5L), 0.0));
list.add(new KeyValue<>(new Edge<>(5L, 4L), 0.0));
Properties producerConfig = ClientUtils.producerConfig(CLUSTER.bootstrapServers(), KryoSerializer.class,
DoubleSerializer.class, new Properties()
);
KTable<Edge<Long>, Double> edges =
StreamUtils.tableFromCollection(builder, producerConfig, new KryoSerde<>(), Serdes.Double(), list);
KGraph<Long, Double, Double> graph = KGraph.fromEdges(edges, new InitVertices(),
GraphSerialized.with(Serdes.Long(), Serdes.Double(), Serdes.Double()));
Properties props = ClientUtils.streamsConfig("prepare-" + suffix, "prepare-client-" + suffix,
CLUSTER.bootstrapServers(), graph.keySerde().getClass(), graph.vertexValueSerde().getClass());
CompletableFuture<Map<TopicPartition, Long>> state = GraphUtils.groupEdgesBySourceAndRepartition(builder, props, graph, "vertices-" + suffix, "edgesGroupedBySource-" + suffix, 2, (short) 1);
Map<TopicPartition, Long> offsets = state.get();
Map<String, Object> configs = new HashMap<>();
configs.put("distance.conversion.enabled", true);
algorithm =
new PregelGraphAlgorithm<>(null, "run-" + suffix, CLUSTER.bootstrapServers(),
CLUSTER.zKConnectString(), "vertices-" + suffix, "edgesGroupedBySource-" + suffix, offsets, graph.serialized(),
"solutionSet-" + suffix, "solutionSetStore-" + suffix, "workSet-" + suffix, 2, (short) 1,
configs, Optional.empty(), new AdamicAdar());
streamsConfiguration = ClientUtils.streamsConfig("run-" + suffix, "run-client-" + suffix,
CLUSTER.bootstrapServers(), graph.keySerde().getClass(), KryoSerde.class);
KafkaStreams streams = algorithm.configure(new StreamsBuilder(), streamsConfiguration).streams();
GraphAlgorithmState<KTable<Long, Double>> paths = algorithm.run();
paths.result().get();
Map<Long, Map<Long, Long>> map = StreamUtils.mapFromStore(paths.streams(), "solutionSetStore-" + suffix);
log.debug("result: {}", map);
assertEquals("{1=-1.0986122886681098, 2=-1.0986122886681098, 3=-0.6931471805599453, 4=-1.3862943611198906, 5=-0.6931471805599453}", map.toString());
Map<Long, Map<Long, Double>> edgesMap = StreamUtils.mapFromStore(paths.streams(), "edgesStore-run-" + suffix);
log.debug("edges : {}", edgesMap);
assertEquals("{1={2=1.3862943611198906, 3=1.3862943611198906, 4=1.791759469228055}, 2={1=1.3862943611198906, 4=1.791759469228055, 5=1.3862943611198906}, 3={1=1.3862943611198906, 4=1.0986122886681098}, 4={1=1.791759469228055, 2=1.791759469228055, 3=1.0986122886681098, 5=1.0986122886681098}, 5={2=1.3862943611198906, 4=1.0986122886681098}}", edgesMap.toString());
}
Example #9
| Source File: JaccardTest.java From kafka-graphs with Apache License 2.0 | 4 votes |
|
@Test
public void testExactSimilarity() throws Exception {
String suffix = "similarity";
StreamsBuilder builder = new StreamsBuilder();
List<KeyValue<Edge<Long>, Double>> list = new ArrayList<>();
list.add(new KeyValue<>(new Edge<>(1L, 2L), 0.0));
list.add(new KeyValue<>(new Edge<>(1L, 3L), 0.0));
list.add(new KeyValue<>(new Edge<>(1L, 4L), 0.0));
list.add(new KeyValue<>(new Edge<>(2L, 1L), 0.0));
list.add(new KeyValue<>(new Edge<>(2L, 4L), 0.0));
list.add(new KeyValue<>(new Edge<>(2L, 5L), 0.0));
list.add(new KeyValue<>(new Edge<>(3L, 1L), 0.0));
list.add(new KeyValue<>(new Edge<>(3L, 4L), 0.0));
list.add(new KeyValue<>(new Edge<>(4L, 1L), 0.0));
list.add(new KeyValue<>(new Edge<>(4L, 2L), 0.0));
list.add(new KeyValue<>(new Edge<>(4L, 3L), 0.0));
list.add(new KeyValue<>(new Edge<>(4L, 5L), 0.0));
list.add(new KeyValue<>(new Edge<>(5L, 2L), 0.0));
list.add(new KeyValue<>(new Edge<>(5L, 4L), 0.0));
list.add(new KeyValue<>(new Edge<>(5L, 6L), 0.0));
list.add(new KeyValue<>(new Edge<>(6L, 5L), 0.0));
Properties producerConfig = ClientUtils.producerConfig(CLUSTER.bootstrapServers(), KryoSerializer.class,
DoubleSerializer.class, new Properties()
);
KTable<Edge<Long>, Double> edges =
StreamUtils.tableFromCollection(builder, producerConfig, new KryoSerde<>(), Serdes.Double(), list);
KGraph<Long, Double, Double> graph = KGraph.fromEdges(edges, new InitVertices(),
GraphSerialized.with(Serdes.Long(), Serdes.Double(), Serdes.Double()));
Properties props = ClientUtils.streamsConfig("prepare-" + suffix, "prepare-client-" + suffix,
CLUSTER.bootstrapServers(), graph.keySerde().getClass(), graph.vertexValueSerde().getClass());
CompletableFuture<Map<TopicPartition, Long>> state = GraphUtils.groupEdgesBySourceAndRepartition(builder, props, graph, "vertices-" + suffix, "edgesGroupedBySource-" + suffix, 2, (short) 1);
Map<TopicPartition, Long> offsets = state.get();
Map<String, Object> configs = new HashMap<>();
configs.put("distance.conversion.enabled", false);
algorithm =
new PregelGraphAlgorithm<>(null, "run-" + suffix, CLUSTER.bootstrapServers(),
CLUSTER.zKConnectString(), "vertices-" + suffix, "edgesGroupedBySource-" + suffix, offsets, graph.serialized(),
"solutionSet-" + suffix, "solutionSetStore-" + suffix, "workSet-" + suffix, 2, (short) 1,
configs, Optional.empty(), new Jaccard<>());
streamsConfiguration = ClientUtils.streamsConfig("run-" + suffix, "run-client-" + suffix,
CLUSTER.bootstrapServers(), graph.keySerde().getClass(), KryoSerde.class);
KafkaStreams streams = algorithm.configure(new StreamsBuilder(), streamsConfiguration).streams();
GraphAlgorithmState<KTable<Long, Double>> paths = algorithm.run();
paths.result().get();
Map<Long, Map<Long, Double>> edgesMap = StreamUtils.mapFromStore(paths.streams(), "edgesStore-run-" + suffix);
log.debug("edges : {}", edgesMap);
assertEquals("{1={2=0.2, 3=0.25, 4=0.4}, 2={1=0.2, 4=0.4, 5=0.2}, 3={1=0.25, 4=0.2}, 4={1=0.4, 2=0.4, 3=0.2, 5=0.16666666666666666}, 5={2=0.2, 4=0.16666666666666666, 6=0.0}, 6={5=0.0}}", edgesMap.toString());
}
Example #10
| Source File: JaccardTest.java From kafka-graphs with Apache License 2.0 | 4 votes |
|
@Test
public void testExactDistance() throws Exception {
String suffix = "distance";
StreamsBuilder builder = new StreamsBuilder();
List<KeyValue<Edge<Long>, Double>> list = new ArrayList<>();
list.add(new KeyValue<>(new Edge<>(1L, 2L), 0.0));
list.add(new KeyValue<>(new Edge<>(1L, 3L), 0.0));
list.add(new KeyValue<>(new Edge<>(1L, 4L), 0.0));
list.add(new KeyValue<>(new Edge<>(2L, 1L), 0.0));
list.add(new KeyValue<>(new Edge<>(2L, 4L), 0.0));
list.add(new KeyValue<>(new Edge<>(2L, 5L), 0.0));
list.add(new KeyValue<>(new Edge<>(3L, 1L), 0.0));
list.add(new KeyValue<>(new Edge<>(3L, 4L), 0.0));
list.add(new KeyValue<>(new Edge<>(4L, 1L), 0.0));
list.add(new KeyValue<>(new Edge<>(4L, 2L), 0.0));
list.add(new KeyValue<>(new Edge<>(4L, 3L), 0.0));
list.add(new KeyValue<>(new Edge<>(4L, 5L), 0.0));
list.add(new KeyValue<>(new Edge<>(5L, 2L), 0.0));
list.add(new KeyValue<>(new Edge<>(5L, 4L), 0.0));
list.add(new KeyValue<>(new Edge<>(5L, 6L), 0.0));
list.add(new KeyValue<>(new Edge<>(6L, 5L), 0.0));
Properties producerConfig = ClientUtils.producerConfig(CLUSTER.bootstrapServers(), KryoSerializer.class,
DoubleSerializer.class, new Properties()
);
KTable<Edge<Long>, Double> edges =
StreamUtils.tableFromCollection(builder, producerConfig, new KryoSerde<>(), Serdes.Double(), list);
KGraph<Long, Double, Double> graph = KGraph.fromEdges(edges, new InitVertices(),
GraphSerialized.with(Serdes.Long(), Serdes.Double(), Serdes.Double()));
Properties props = ClientUtils.streamsConfig("prepare-" + suffix, "prepare-client-" + suffix,
CLUSTER.bootstrapServers(), graph.keySerde().getClass(), graph.vertexValueSerde().getClass());
CompletableFuture<Map<TopicPartition, Long>> state = GraphUtils.groupEdgesBySourceAndRepartition(builder, props, graph, "vertices-" + suffix, "edgesGroupedBySource-" + suffix, 2, (short) 1);
Map<TopicPartition, Long> offsets = state.get();
Map<String, Object> configs = new HashMap<>();
configs.put("distance.conversion.enabled", true);
algorithm =
new PregelGraphAlgorithm<>(null, "run-" + suffix, CLUSTER.bootstrapServers(),
CLUSTER.zKConnectString(), "vertices-" + suffix, "edgesGroupedBySource-" + suffix, offsets, graph.serialized(),
"solutionSet-" + suffix, "solutionSetStore-" + suffix, "workSet-" + suffix, 2, (short) 1,
configs, Optional.empty(), new Jaccard<>());
streamsConfiguration = ClientUtils.streamsConfig("run-" + suffix, "run-client-" + suffix,
CLUSTER.bootstrapServers(), graph.keySerde().getClass(), KryoSerde.class);
KafkaStreams streams = algorithm.configure(new StreamsBuilder(), streamsConfiguration).streams();
GraphAlgorithmState<KTable<Long, Double>> paths = algorithm.run();
paths.result().get();
Thread.sleep(2000);
Map<Long, Map<Long, Double>> edgesMap = StreamUtils.mapFromStore(paths.streams(), "edgesStore-run-" + suffix);
log.debug("edges : {}", edgesMap);
assertEquals("{1={2=4.0, 3=3.0, 4=1.5}, 2={1=4.0, 4=1.5, 5=4.0}, 3={1=3.0, 4=4.0}, 4={1=1.5, 2=1.5, 3=4.0, 5=5.0}, 5={2=4.0, 4=5.0, 6=1.7976931348623157E308}, 6={5=1.7976931348623157E308}}", edgesMap.toString());
}
Example #11
| Source File: SemiClusteringTest.java From kafka-graphs with Apache License 2.0 | 4 votes |
|
@Test
public void testSemiClustering() throws Exception {
String suffix = "";
StreamsBuilder builder = new StreamsBuilder();
List<KeyValue<Edge<Long>, Double>> list = new ArrayList<>();
list.add(new KeyValue<>(new Edge<>(1L, 2L), 1.0));
list.add(new KeyValue<>(new Edge<>(2L, 1L), 1.0));
list.add(new KeyValue<>(new Edge<>(1L, 3L), 1.0));
list.add(new KeyValue<>(new Edge<>(3L, 1L), 1.0));
list.add(new KeyValue<>(new Edge<>(2L, 3L), 2.0));
list.add(new KeyValue<>(new Edge<>(3L, 2L), 2.0));
list.add(new KeyValue<>(new Edge<>(3L, 4L), 2.0));
list.add(new KeyValue<>(new Edge<>(4L, 3L), 2.0));
list.add(new KeyValue<>(new Edge<>(3L, 5L), 1.0));
list.add(new KeyValue<>(new Edge<>(5L, 3L), 1.0));
list.add(new KeyValue<>(new Edge<>(4L, 5L), 1.0));
list.add(new KeyValue<>(new Edge<>(5L, 4L), 1.0));
Properties producerConfig = ClientUtils.producerConfig(CLUSTER.bootstrapServers(), KryoSerializer.class,
DoubleSerializer.class, new Properties()
);
KTable<Edge<Long>, Double> edges =
StreamUtils.tableFromCollection(builder, producerConfig, new KryoSerde<>(), Serdes.Double(), list);
KGraph<Long, Set<SemiCluster>, Double> graph = KGraph.fromEdges(edges, new InitVertices(),
GraphSerialized.with(Serdes.Long(), new KryoSerde<>(), Serdes.Double()));
Properties props = ClientUtils.streamsConfig("prepare-" + suffix, "prepare-client-" + suffix,
CLUSTER.bootstrapServers(), graph.keySerde().getClass(), graph.vertexValueSerde().getClass());
CompletableFuture<Map<TopicPartition, Long>> state = GraphUtils.groupEdgesBySourceAndRepartition(builder, props, graph, "vertices-" + suffix, "edgesGroupedBySource-" + suffix, 2, (short) 1);
Map<TopicPartition, Long> offsets = state.get();
Map<String, Object> configs = new HashMap<>();
configs.put(SemiClustering.ITERATIONS, 10);
configs.put(SemiClustering.MAX_CLUSTERS, 2);
configs.put(SemiClustering.CLUSTER_CAPACITY, 2);
algorithm =
new PregelGraphAlgorithm<>(null, "run-" + suffix, CLUSTER.bootstrapServers(),
CLUSTER.zKConnectString(), "vertices-" + suffix, "edgesGroupedBySource-" + suffix, offsets, graph.serialized(),
"solutionSet-" + suffix, "solutionSetStore-" + suffix, "workSet-" + suffix, 2, (short) 1,
configs, Optional.empty(), new SemiClustering());
streamsConfiguration = ClientUtils.streamsConfig("run-" + suffix, "run-client-" + suffix,
CLUSTER.bootstrapServers(), graph.keySerde().getClass(), KryoSerde.class);
KafkaStreams streams = algorithm.configure(new StreamsBuilder(), streamsConfiguration).streams();
GraphAlgorithmState<KTable<Long, Set<SemiCluster>>> paths = algorithm.run();
paths.result().get();
Thread.sleep(2000);
Map<Long, Map<Long, Long>> map = StreamUtils.mapFromStore(paths.streams(), "solutionSetStore-" + suffix);
log.debug("result: {}", map);
assertEquals("{1=[[ 1 4 | -2.5, 0.0, 5.0 ], [ 1 3 | -2.0, 1.0, 6.0 ]], 2=[[ 2 4 | -3.0, 0.0, 6.0 ], [ 2 3 | -0.5, 2.0, 5.0 ]], 3=[[ 3 4 | -0.5, 2.0, 5.0 ], [ 3 | 0.0, 0.0, 6.0 ]], 4=[[ 3 4 | -0.5, 2.0, 5.0 ], [ 4 | 0.0, 0.0, 3.0 ]], 5=[[ 3 5 | -2.0, 1.0, 6.0 ], [ 4 5 | -0.5, 1.0, 3.0 ]]}", map.toString());
}
Example #12
| Source File: SybilRankTest.java From kafka-graphs with Apache License 2.0 | 4 votes |
|
@Test
public void testSybilRank() throws Exception {
String suffix = "";
StreamsBuilder builder = new StreamsBuilder();
List<KeyValue<Edge<Long>, Double>> list = new ArrayList<>();
list.add(new KeyValue<>(new Edge<>(1L, 2L), 5.0));
list.add(new KeyValue<>(new Edge<>(2L, 1L), 5.0));
list.add(new KeyValue<>(new Edge<>(2L, 4L), 4.0));
list.add(new KeyValue<>(new Edge<>(4L, 2L), 4.0));
list.add(new KeyValue<>(new Edge<>(4L, 5L), 3.0));
list.add(new KeyValue<>(new Edge<>(5L, 4L), 3.0));
list.add(new KeyValue<>(new Edge<>(3L, 5L), 3.0));
list.add(new KeyValue<>(new Edge<>(5L, 3L), 3.0));
list.add(new KeyValue<>(new Edge<>(1L, 3L), 2.0));
list.add(new KeyValue<>(new Edge<>(3L, 1L), 2.0));
list.add(new KeyValue<>(new Edge<>(3L, 7L), 1.0));
list.add(new KeyValue<>(new Edge<>(7L, 3L), 1.0));
list.add(new KeyValue<>(new Edge<>(6L, 7L), 3.0));
list.add(new KeyValue<>(new Edge<>(7L, 6L), 3.0));
list.add(new KeyValue<>(new Edge<>(6L, 9L), 3.0));
list.add(new KeyValue<>(new Edge<>(9L, 6L), 3.0));
list.add(new KeyValue<>(new Edge<>(8L, 9L), 2.0));
list.add(new KeyValue<>(new Edge<>(9L, 8L), 2.0));
list.add(new KeyValue<>(new Edge<>(7L, 8L), 3.0));
list.add(new KeyValue<>(new Edge<>(8L, 7L), 3.0));
Properties producerConfig = ClientUtils.producerConfig(CLUSTER.bootstrapServers(), KryoSerializer.class,
DoubleSerializer.class, new Properties()
);
KTable<Edge<Long>, Double> edges =
StreamUtils.tableFromCollection(builder, producerConfig, new KryoSerde<>(), Serdes.Double(), list);
KGraph<Long, VertexValue, Double> graph = KGraph.fromEdges(edges, new InitVertices(),
GraphSerialized.with(Serdes.Long(), new KryoSerde<>(), Serdes.Double()));
Properties props = ClientUtils.streamsConfig("prepare-" + suffix, "prepare-client-" + suffix,
CLUSTER.bootstrapServers(), graph.keySerde().getClass(), graph.vertexValueSerde().getClass());
CompletableFuture<Map<TopicPartition, Long>> state = GraphUtils.groupEdgesBySourceAndRepartition(builder, props, graph, "vertices-" + suffix, "edgesGroupedBySource-" + suffix, 2, (short) 1);
Map<TopicPartition, Long> offsets = state.get();
Map<String, Object> configs = new HashMap<>();
algorithm =
new PregelGraphAlgorithm<>(null, "run-" + suffix, CLUSTER.bootstrapServers(),
CLUSTER.zKConnectString(), "vertices-" + suffix, "edgesGroupedBySource-" + suffix, offsets, graph.serialized(),
"solutionSet-" + suffix, "solutionSetStore-" + suffix, "workSet-" + suffix, 2, (short) 1,
configs, Optional.empty(), new SybilRank());
streamsConfiguration = ClientUtils.streamsConfig("run-" + suffix, "run-client-" + suffix,
CLUSTER.bootstrapServers(), graph.keySerde().getClass(), KryoSerde.class);
KafkaStreams streams = algorithm.configure(new StreamsBuilder(), streamsConfiguration).streams();
GraphAlgorithmState<KTable<Long, VertexValue>> paths = algorithm.run();
paths.result().get();
Thread.sleep(2000);
Map<Long, Map<Long, Long>> map = StreamUtils.mapFromStore(paths.streams(), "solutionSetStore-" + suffix);
log.debug("result: {}", map);
assertEquals("{1=0.2380952380952381, 2=0.23809523809523808, 3=0.39285714285714285, 4=0.4047619047619047, 5=0.0, 6=0.0, 7=0.0, 8=0.0, 9=0.0}", map.toString());
}
Example #13
| Source File: LocalClusteringCoefficientTest.java From kafka-graphs with Apache License 2.0 | 4 votes |
|
@Test
public void testLCC() throws Exception {
String suffix = "";
StreamsBuilder builder = new StreamsBuilder();
Properties producerConfig = ClientUtils.producerConfig(CLUSTER.bootstrapServers(), LongSerializer.class,
DoubleSerializer.class, new Properties()
);
KTable<Edge<Long>, Double> edges =
StreamUtils.tableFromCollection(builder, producerConfig, new KryoSerde<>(), Serdes.Double(),
TestGraphUtils.getLCCEdges());
KGraph<Long, Double, Double> graph = KGraph.fromEdges(edges,
GraphAlgorithmType.initialVertexValueMapper(GraphAlgorithmType.lcc),
GraphSerialized.with(Serdes.Long(), Serdes.Double(), Serdes.Double()));
Properties props = ClientUtils.streamsConfig("prepare", "prepare-client", CLUSTER.bootstrapServers(),
graph.keySerde().getClass(), graph.vertexValueSerde().getClass());
CompletableFuture<Map<TopicPartition, Long>> state = GraphUtils.groupEdgesBySourceAndRepartition(builder, props, graph, "vertices-" + suffix, "edgesGroupedBySource-" + suffix, 2, (short) 1);
Map<TopicPartition, Long> offsets = state.get();
algorithm =
new PregelGraphAlgorithm<>(null, "run", CLUSTER.bootstrapServers(),
CLUSTER.zKConnectString(), "vertices-" + suffix, "edgesGroupedBySource-" + suffix, offsets, graph.serialized(),
"solutionSet", "solutionSetStore", "workSet", 2, (short) 1,
Collections.emptyMap(), Optional.empty(), new LocalClusteringCoefficient());
props = ClientUtils.streamsConfig("run", "run-client", CLUSTER.bootstrapServers(),
graph.keySerde().getClass(), KryoSerde.class);
KafkaStreams streams = algorithm.configure(new StreamsBuilder(), props).streams();
GraphAlgorithmState<KTable<Long, Double>> paths = algorithm.run();
paths.result().get();
Map<Long, Double> map = StreamUtils.mapFromStore(paths.streams(), "solutionSetStore");
log.debug("result: {}", map);
Map<Long, Double> expectedResult = new HashMap<>();
expectedResult.put(0L, 0.5);
expectedResult.put(1L, 0.3333333333333333);
expectedResult.put(2L, 0.3333333333333333);
expectedResult.put(3L, 0.08333333333333333);
expectedResult.put(4L, 0.0);
expectedResult.put(5L, 0.0);
assertEquals(expectedResult, map);
}
Example #14
| Source File: SingleSourceShortestPathsTest.java From kafka-graphs with Apache License 2.0 | 4 votes |
|
@Test
public void testSingleSourceShortestPaths() throws Exception {
String suffix = "";
StreamsBuilder builder = new StreamsBuilder();
Properties producerConfig = ClientUtils.producerConfig(CLUSTER.bootstrapServers(), LongSerializer.class,
DoubleSerializer.class, new Properties()
);
KTable<Edge<Long>, Double> edges =
StreamUtils.tableFromCollection(builder, producerConfig, new KryoSerde<>(), Serdes.Double(),
TestGraphUtils.getLongDoubleEdges());
KGraph<Long, Double, Double> graph = KGraph.fromEdges(edges,
GraphAlgorithmType.initialVertexValueMapper(GraphAlgorithmType.sssp),
GraphSerialized.with(Serdes.Long(), Serdes.Double(), Serdes.Double()));
Properties props = ClientUtils.streamsConfig("prepare", "prepare-client", CLUSTER.bootstrapServers(),
graph.keySerde().getClass(), graph.vertexValueSerde().getClass());
CompletableFuture<Map<TopicPartition, Long>> state = GraphUtils.groupEdgesBySourceAndRepartition(builder, props, graph, "vertices-" + suffix, "edgesGroupedBySource-" + suffix, 2, (short) 1);
Map<TopicPartition, Long> offsets = state.get();
Map<String, Object> configs = new HashMap<>();
configs.put(SingleSourceShortestPaths.SRC_VERTEX_ID, 1L);
algorithm =
new PregelGraphAlgorithm<>(null, "run", CLUSTER.bootstrapServers(),
CLUSTER.zKConnectString(), "vertices-" + suffix, "edgesGroupedBySource-" + suffix, offsets, graph.serialized(),
"solutionSet", "solutionSetStore", "workSet", 2, (short) 1,
configs, Optional.empty(), new SingleSourceShortestPaths());
props = ClientUtils.streamsConfig("run", "run-client", CLUSTER.bootstrapServers(),
graph.keySerde().getClass(), KryoSerde.class);
KafkaStreams streams = algorithm.configure(new StreamsBuilder(), props).streams();
GraphAlgorithmState<KTable<Long, Double>> paths = algorithm.run();
paths.result().get();
Map<Long, Double> map = StreamUtils.mapFromStore(paths.streams(), "solutionSetStore");
log.debug("result: {}", map);
Map<Long, Double> expectedResult = new HashMap<>();
expectedResult.put(1L, 0.0);
expectedResult.put(2L, 12.0);
expectedResult.put(3L, 13.0);
expectedResult.put(4L, 47.0);
expectedResult.put(5L, 48.0);
assertEquals(expectedResult, map);
}
Example #15
| Source File: MultipleSourceShortestPathsTest.java From kafka-graphs with Apache License 2.0 | 4 votes |
|
@Test
public void testMultipleSourceShortestPaths() throws Exception {
String suffix = "";
StreamsBuilder builder = new StreamsBuilder();
List<KeyValue<Edge<Long>, Double>> edges = new ArrayList<>();
edges.add(new KeyValue<>(new Edge<>(1L, 2L), 1.0));
edges.add(new KeyValue<>(new Edge<>(1L, 5L), 1.0));
edges.add(new KeyValue<>(new Edge<>(2L, 3L), 1.0));
edges.add(new KeyValue<>(new Edge<>(2L, 5L), 1.0));
edges.add(new KeyValue<>(new Edge<>(3L, 4L), 1.0));
edges.add(new KeyValue<>(new Edge<>(4L, 5L), 1.0));
edges.add(new KeyValue<>(new Edge<>(4L, 6L), 1.0));
edges.add(new KeyValue<>(new Edge<>(2L, 1L), 1.0));
edges.add(new KeyValue<>(new Edge<>(5L, 1L), 1.0));
edges.add(new KeyValue<>(new Edge<>(3L, 2L), 1.0));
edges.add(new KeyValue<>(new Edge<>(5L, 2L), 1.0));
edges.add(new KeyValue<>(new Edge<>(4L, 3L), 1.0));
edges.add(new KeyValue<>(new Edge<>(5L, 4L), 1.0));
edges.add(new KeyValue<>(new Edge<>(6L, 4L), 1.0));
Properties producerConfig = ClientUtils.producerConfig(CLUSTER.bootstrapServers(), LongSerializer.class,
DoubleSerializer.class, new Properties()
);
KTable<Edge<Long>, Double> table =
StreamUtils.tableFromCollection(builder, producerConfig, new KryoSerde<>(), Serdes.Double(),
edges);
KGraph<Long, Map<Long, Double>, Double> graph = KGraph.fromEdges(table,
GraphAlgorithmType.initialVertexValueMapper(GraphAlgorithmType.mssp),
GraphSerialized.with(Serdes.Long(), new KryoSerde<>(), Serdes.Double()));
Properties props = ClientUtils.streamsConfig("prepare", "prepare-client", CLUSTER.bootstrapServers(),
graph.keySerde().getClass(), graph.vertexValueSerde().getClass());
CompletableFuture<Map<TopicPartition, Long>> state = GraphUtils.groupEdgesBySourceAndRepartition(builder, props, graph, "vertices-" + suffix, "edgesGroupedBySource-" + suffix, 2, (short) 1);
Map<TopicPartition, Long> offsets = state.get();
Set<Long> landmarks = new HashSet<>();
landmarks.add(1L);
landmarks.add(4L);
Map<String, Object> configs = new HashMap<>();
configs.put(MultipleSourceShortestPaths.LANDMARK_VERTEX_IDS, landmarks);
algorithm =
new PregelGraphAlgorithm<>(null, "run", CLUSTER.bootstrapServers(),
CLUSTER.zKConnectString(), "vertices-" + suffix, "edgesGroupedBySource-" + suffix, offsets, graph.serialized(),
"solutionSet", "solutionSetStore", "workSet", 2, (short) 1,
configs, Optional.empty(), new MultipleSourceShortestPaths());
props = ClientUtils.streamsConfig("run", "run-client", CLUSTER.bootstrapServers(),
graph.keySerde().getClass(), KryoSerde.class);
KafkaStreams streams = algorithm.configure(new StreamsBuilder(), props).streams();
GraphAlgorithmState<KTable<Long, Map<Long, Double>>> paths = algorithm.run();
paths.result().get();
Map<Long, Map<Long, Double>> map = StreamUtils.mapFromStore(paths.streams(), "solutionSetStore");
log.debug("result: {}", map);
Map<Long, Map<Long, Double>> expectedResult = new HashMap<>();
expectedResult.put(1L, new HashMap<Long, Double>() {{
put(1L, 0.0);
put(4L, 2.0);
}});
expectedResult.put(2L, new HashMap<Long, Double>() {{
put(1L, 1.0);
put(4L, 2.0);
}});
expectedResult.put(3L, new HashMap<Long, Double>() {{
put(1L, 2.0);
put(4L, 1.0);
}});
expectedResult.put(4L, new HashMap<Long, Double>() {{
put(1L, 2.0);
put(4L, 0.0);
}});
expectedResult.put(5L, new HashMap<Long, Double>() {{
put(1L, 1.0);
put(4L, 1.0);
}});
expectedResult.put(6L, new HashMap<Long, Double>() {{
put(1L, 3.0);
put(4L, 1.0);
}});
assertEquals(expectedResult, map);
}
