org.apache.flink.api.common.functions.RichFlatMapFunction Java Examples
The following examples show how to use
org.apache.flink.api.common.functions.RichFlatMapFunction.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
| Source File: TypeExtractorTest.java From flink with Apache License 2.0 | 6 votes |
|
@SuppressWarnings({ "unchecked", "rawtypes" })
@Test
public void testTuple0() {
// use getFlatMapReturnTypes()
RichFlatMapFunction<?, ?> function = new RichFlatMapFunction<Tuple0, Tuple0>() {
private static final long serialVersionUID = 1L;
@Override
public void flatMap(Tuple0 value, Collector<Tuple0> out) throws Exception {
// nothing to do
}
};
TypeInformation<?> ti = TypeExtractor.getFlatMapReturnTypes(function,
(TypeInformation) TypeInformation.of(new TypeHint<Tuple0>(){}));
Assert.assertTrue(ti.isTupleType());
Assert.assertEquals(0, ti.getArity());
Assert.assertTrue(ti instanceof TupleTypeInfo);
}
Example #2
| Source File: TypeExtractorTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
|
@SuppressWarnings({ "unchecked", "rawtypes" })
@Test
public void testTuple0() {
// use getFlatMapReturnTypes()
RichFlatMapFunction<?, ?> function = new RichFlatMapFunction<Tuple0, Tuple0>() {
private static final long serialVersionUID = 1L;
@Override
public void flatMap(Tuple0 value, Collector<Tuple0> out) throws Exception {
// nothing to do
}
};
TypeInformation<?> ti = TypeExtractor.getFlatMapReturnTypes(function,
(TypeInformation) TypeInformation.of(new TypeHint<Tuple0>(){}));
Assert.assertTrue(ti.isTupleType());
Assert.assertEquals(0, ti.getArity());
Assert.assertTrue(ti instanceof TupleTypeInfo);
}
Example #3
| Source File: TypeExtractorTest.java From flink with Apache License 2.0 | 6 votes |
|
@SuppressWarnings({ "unchecked", "rawtypes" })
@Test
public void testTuple0() {
// use getFlatMapReturnTypes()
RichFlatMapFunction<?, ?> function = new RichFlatMapFunction<Tuple0, Tuple0>() {
private static final long serialVersionUID = 1L;
@Override
public void flatMap(Tuple0 value, Collector<Tuple0> out) throws Exception {
// nothing to do
}
};
TypeInformation<?> ti = TypeExtractor.getFlatMapReturnTypes(function,
(TypeInformation) TypeInformation.of(new TypeHint<Tuple0>(){}));
Assert.assertTrue(ti.isTupleType());
Assert.assertEquals(0, ti.getArity());
Assert.assertTrue(ti instanceof TupleTypeInfo);
}
Example #4
| Source File: MiscellaneousIssuesITCase.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
@Test
public void testAccumulatorsAfterNoOp() {
final String accName = "test_accumulator";
try {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(6);
env.getConfig().disableSysoutLogging();
env.generateSequence(1, 1000000)
.rebalance()
.flatMap(new RichFlatMapFunction<Long, Long>() {
private LongCounter counter;
@Override
public void open(Configuration parameters) {
counter = getRuntimeContext().getLongCounter(accName);
}
@Override
public void flatMap(Long value, Collector<Long> out) {
counter.add(1L);
}
})
.output(new DiscardingOutputFormat<Long>());
JobExecutionResult result = env.execute();
assertEquals(1000000L, result.getAllAccumulatorResults().get(accName));
}
catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
Example #5
| Source File: MiscellaneousIssuesITCase.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testAccumulatorsAfterNoOp() {
final String accName = "test_accumulator";
try {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(6);
env.generateSequence(1, 1000000)
.rebalance()
.flatMap(new RichFlatMapFunction<Long, Long>() {
private LongCounter counter;
@Override
public void open(Configuration parameters) {
counter = getRuntimeContext().getLongCounter(accName);
}
@Override
public void flatMap(Long value, Collector<Long> out) {
counter.add(1L);
}
})
.output(new DiscardingOutputFormat<Long>());
JobExecutionResult result = env.execute();
assertEquals(1000000L, result.getAllAccumulatorResults().get(accName));
}
catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
Example #6
| Source File: MiscellaneousIssuesITCase.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testAccumulatorsAfterNoOp() {
final String accName = "test_accumulator";
try {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(6);
env.getConfig().disableSysoutLogging();
env.generateSequence(1, 1000000)
.rebalance()
.flatMap(new RichFlatMapFunction<Long, Long>() {
private LongCounter counter;
@Override
public void open(Configuration parameters) {
counter = getRuntimeContext().getLongCounter(accName);
}
@Override
public void flatMap(Long value, Collector<Long> out) {
counter.add(1L);
}
})
.output(new DiscardingOutputFormat<Long>());
JobExecutionResult result = env.execute();
assertEquals(1000000L, result.getAllAccumulatorResults().get(accName));
}
catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
Example #7
| Source File: FunctionCompiler.java From rheem with Apache License 2.0 | 5 votes |
|
public <I, O> RichFlatMapFunction<I, O> compile(FunctionDescriptor.ExtendedSerializableFunction<I, Iterable<O>> flatMapDescriptor, FlinkExecutionContext exe) {
return new RichFlatMapFunction<I, O>() {
@Override
public void open(Configuration parameters) throws Exception {
flatMapDescriptor.open(exe);
}
@Override
public void flatMap(I value, Collector<O> out) throws Exception {
flatMapDescriptor.apply(value).forEach(out::collect);
}
};
}
Example #8
| Source File: TypeExtractorTest.java From flink with Apache License 2.0 | 4 votes |
|
@SuppressWarnings({ "unchecked", "rawtypes" })
@Test
public void testTupleWithTuples() {
// use getFlatMapReturnTypes()
RichFlatMapFunction<?, ?> function = new RichFlatMapFunction<Tuple3<Tuple1<String>, Tuple1<Integer>, Tuple2<Long, Long>>, Tuple3<Tuple1<String>, Tuple1<Integer>, Tuple2<Long, Long>>>() {
private static final long serialVersionUID = 1L;
@Override
public void flatMap(Tuple3<Tuple1<String>, Tuple1<Integer>, Tuple2<Long, Long>> value,
Collector<Tuple3<Tuple1<String>, Tuple1<Integer>, Tuple2<Long, Long>>> out) throws Exception {
// nothing to do
}
};
TypeInformation<?> ti = TypeExtractor.getFlatMapReturnTypes(function, (TypeInformation) TypeInformation.of(new TypeHint<Tuple3<Tuple1<String>, Tuple1<Integer>, Tuple2<Long, Long>>>(){}));
Assert.assertTrue(ti.isTupleType());
Assert.assertEquals(3, ti.getArity());
Assert.assertTrue(ti instanceof TupleTypeInfo);
List<FlatFieldDescriptor> ffd = new ArrayList<FlatFieldDescriptor>();
((TupleTypeInfo) ti).getFlatFields("f0.f0", 0, ffd);
Assert.assertEquals(0, ffd.get(0).getPosition() );
ffd.clear();
((TupleTypeInfo) ti).getFlatFields("f0.f0", 0, ffd);
Assert.assertTrue( ffd.get(0).getType() instanceof BasicTypeInfo );
Assert.assertTrue( ffd.get(0).getType().getTypeClass().equals(String.class) );
ffd.clear();
((TupleTypeInfo) ti).getFlatFields("f1.f0", 0, ffd);
Assert.assertEquals(1, ffd.get(0).getPosition() );
ffd.clear();
TupleTypeInfo<?> tti = (TupleTypeInfo<?>) ti;
Assert.assertEquals(Tuple3.class, tti.getTypeClass());
Assert.assertTrue(tti.getTypeAt(0).isTupleType());
Assert.assertTrue(tti.getTypeAt(1).isTupleType());
Assert.assertTrue(tti.getTypeAt(2).isTupleType());
Assert.assertEquals(Tuple1.class, tti.getTypeAt(0).getTypeClass());
Assert.assertEquals(Tuple1.class, tti.getTypeAt(1).getTypeClass());
Assert.assertEquals(Tuple2.class, tti.getTypeAt(2).getTypeClass());
Assert.assertEquals(1, tti.getTypeAt(0).getArity());
Assert.assertEquals(1, tti.getTypeAt(1).getArity());
Assert.assertEquals(2, tti.getTypeAt(2).getArity());
Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO, ((TupleTypeInfo<?>) tti.getTypeAt(0)).getTypeAt(0));
Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO, ((TupleTypeInfo<?>) tti.getTypeAt(1)).getTypeAt(0));
Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO, ((TupleTypeInfo<?>) tti.getTypeAt(2)).getTypeAt(0));
Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO, ((TupleTypeInfo<?>) tti.getTypeAt(2)).getTypeAt(1));
// use getForObject()
Tuple3<Tuple1<String>, Tuple1<Integer>, Tuple2<Long, Long>> t = new Tuple3<Tuple1<String>, Tuple1<Integer>, Tuple2<Long, Long>>(
new Tuple1<String>("hello"), new Tuple1<Integer>(1), new Tuple2<Long, Long>(2L, 3L));
Assert.assertTrue(TypeExtractor.getForObject(t) instanceof TupleTypeInfo);
TupleTypeInfo<?> tti2 = (TupleTypeInfo<?>) TypeExtractor.getForObject(t);
Assert.assertEquals(1, tti2.getTypeAt(0).getArity());
Assert.assertEquals(1, tti2.getTypeAt(1).getArity());
Assert.assertEquals(2, tti2.getTypeAt(2).getArity());
Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO, ((TupleTypeInfo<?>) tti2.getTypeAt(0)).getTypeAt(0));
Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO, ((TupleTypeInfo<?>) tti2.getTypeAt(1)).getTypeAt(0));
Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO, ((TupleTypeInfo<?>) tti2.getTypeAt(2)).getTypeAt(0));
Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO, ((TupleTypeInfo<?>) tti2.getTypeAt(2)).getTypeAt(1));
}
Example #9
| Source File: ChainingSpeed.java From flink-perf with Apache License 2.0 | 4 votes |
|
public static void main(String[] args) throws Exception {
StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
final ParameterTool pt = ParameterTool.fromArgs(args);
see.getConfig().setGlobalJobParameters(pt);
see.getConfig().enableObjectReuse();
// see.setParallelism(1);
DataStreamSource<Integer> src = see.addSource(new RichParallelSourceFunction<Integer>() {
boolean running = true;
@Override
public void run(SourceContext<Integer> ctx) throws Exception {
int i = 0;
while (running) {
ctx.collect(i++);
}
}
@Override
public void cancel() {
running = false;
}
});
src/*.map(new MapFunction<Integer, Integer>() {
@Override
public Integer map(Integer s) throws Exception {
return s;
}
}).*/.map(new MapFunction<Integer, Integer>() {
@Override
public Integer map(Integer s) throws Exception {
return s;
}
}).flatMap(new RichFlatMapFunction<Integer, Integer>() {
long received = 0;
long logfreq = pt.getInt("logfreq");
long lastLog = -1;
long lastElements = 0;
long matches = 0;
private final Pattern threeDigitAbbr = Pattern.compile("[A-Z]{3}\\.");
@Override
public void open(Configuration parameters) throws Exception {
super.open(parameters);
}
@Override
public void flatMap(Integer in, Collector<Integer> collector) throws Exception {
received++;
if (received % logfreq == 0) {
// throughput over entire time
long now = System.currentTimeMillis();
// throughput for the last "logfreq" elements
if (lastLog == -1) {
// init (the first)
lastLog = now;
lastElements = received;
} else {
long timeDiff = now - lastLog;
long elementDiff = received - lastElements;
double ex = (1000 / (double) timeDiff);
LOG.info("During the last {} ms, we received {} elements. That's {} elements/second/core", timeDiff, elementDiff, Double.valueOf(elementDiff * ex).longValue());
// reinit
lastLog = now;
lastElements = received;
}
}
}
});
see.execute();
}
Example #10
| Source File: TypeExtractorTest.java From flink with Apache License 2.0 | 4 votes |
|
@SuppressWarnings({ "unchecked", "rawtypes" })
@Test
public void testTupleWithTuples() {
// use getFlatMapReturnTypes()
RichFlatMapFunction<?, ?> function = new RichFlatMapFunction<Tuple3<Tuple1<String>, Tuple1<Integer>, Tuple2<Long, Long>>, Tuple3<Tuple1<String>, Tuple1<Integer>, Tuple2<Long, Long>>>() {
private static final long serialVersionUID = 1L;
@Override
public void flatMap(Tuple3<Tuple1<String>, Tuple1<Integer>, Tuple2<Long, Long>> value,
Collector<Tuple3<Tuple1<String>, Tuple1<Integer>, Tuple2<Long, Long>>> out) throws Exception {
// nothing to do
}
};
TypeInformation<?> ti = TypeExtractor.getFlatMapReturnTypes(function, (TypeInformation) TypeInformation.of(new TypeHint<Tuple3<Tuple1<String>, Tuple1<Integer>, Tuple2<Long, Long>>>(){}));
Assert.assertTrue(ti.isTupleType());
Assert.assertEquals(3, ti.getArity());
Assert.assertTrue(ti instanceof TupleTypeInfo);
List<FlatFieldDescriptor> ffd = new ArrayList<FlatFieldDescriptor>();
((TupleTypeInfo) ti).getFlatFields("f0.f0", 0, ffd);
Assert.assertEquals(0, ffd.get(0).getPosition() );
ffd.clear();
((TupleTypeInfo) ti).getFlatFields("f0.f0", 0, ffd);
Assert.assertTrue( ffd.get(0).getType() instanceof BasicTypeInfo );
Assert.assertTrue( ffd.get(0).getType().getTypeClass().equals(String.class) );
ffd.clear();
((TupleTypeInfo) ti).getFlatFields("f1.f0", 0, ffd);
Assert.assertEquals(1, ffd.get(0).getPosition() );
ffd.clear();
TupleTypeInfo<?> tti = (TupleTypeInfo<?>) ti;
Assert.assertEquals(Tuple3.class, tti.getTypeClass());
Assert.assertTrue(tti.getTypeAt(0).isTupleType());
Assert.assertTrue(tti.getTypeAt(1).isTupleType());
Assert.assertTrue(tti.getTypeAt(2).isTupleType());
Assert.assertEquals(Tuple1.class, tti.getTypeAt(0).getTypeClass());
Assert.assertEquals(Tuple1.class, tti.getTypeAt(1).getTypeClass());
Assert.assertEquals(Tuple2.class, tti.getTypeAt(2).getTypeClass());
Assert.assertEquals(1, tti.getTypeAt(0).getArity());
Assert.assertEquals(1, tti.getTypeAt(1).getArity());
Assert.assertEquals(2, tti.getTypeAt(2).getArity());
Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO, ((TupleTypeInfo<?>) tti.getTypeAt(0)).getTypeAt(0));
Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO, ((TupleTypeInfo<?>) tti.getTypeAt(1)).getTypeAt(0));
Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO, ((TupleTypeInfo<?>) tti.getTypeAt(2)).getTypeAt(0));
Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO, ((TupleTypeInfo<?>) tti.getTypeAt(2)).getTypeAt(1));
// use getForObject()
Tuple3<Tuple1<String>, Tuple1<Integer>, Tuple2<Long, Long>> t = new Tuple3<Tuple1<String>, Tuple1<Integer>, Tuple2<Long, Long>>(
new Tuple1<String>("hello"), new Tuple1<Integer>(1), new Tuple2<Long, Long>(2L, 3L));
Assert.assertTrue(TypeExtractor.getForObject(t) instanceof TupleTypeInfo);
TupleTypeInfo<?> tti2 = (TupleTypeInfo<?>) TypeExtractor.getForObject(t);
Assert.assertEquals(1, tti2.getTypeAt(0).getArity());
Assert.assertEquals(1, tti2.getTypeAt(1).getArity());
Assert.assertEquals(2, tti2.getTypeAt(2).getArity());
Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO, ((TupleTypeInfo<?>) tti2.getTypeAt(0)).getTypeAt(0));
Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO, ((TupleTypeInfo<?>) tti2.getTypeAt(1)).getTypeAt(0));
Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO, ((TupleTypeInfo<?>) tti2.getTypeAt(2)).getTypeAt(0));
Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO, ((TupleTypeInfo<?>) tti2.getTypeAt(2)).getTypeAt(1));
}
Example #11
| Source File: StatefulJobSavepointMigrationITCase.java From flink with Apache License 2.0 | 4 votes |
|
@Test
public void testSavepoint() throws Exception {
final int parallelism = 4;
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setRestartStrategy(RestartStrategies.noRestart());
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
switch (testStateBackend) {
case StateBackendLoader.ROCKSDB_STATE_BACKEND_NAME:
env.setStateBackend(new RocksDBStateBackend(new MemoryStateBackend()));
break;
case StateBackendLoader.MEMORY_STATE_BACKEND_NAME:
env.setStateBackend(new MemoryStateBackend());
break;
default:
throw new UnsupportedOperationException();
}
env.enableCheckpointing(500);
env.setParallelism(parallelism);
env.setMaxParallelism(parallelism);
SourceFunction<Tuple2<Long, Long>> nonParallelSource;
SourceFunction<Tuple2<Long, Long>> parallelSource;
RichFlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> flatMap;
OneInputStreamOperator<Tuple2<Long, Long>, Tuple2<Long, Long>> timelyOperator;
if (executionMode == ExecutionMode.PERFORM_SAVEPOINT) {
nonParallelSource = new MigrationTestUtils.CheckpointingNonParallelSourceWithListState(NUM_SOURCE_ELEMENTS);
parallelSource = new MigrationTestUtils.CheckpointingParallelSourceWithUnionListState(NUM_SOURCE_ELEMENTS);
flatMap = new CheckpointingKeyedStateFlatMap();
timelyOperator = new CheckpointingTimelyStatefulOperator();
} else if (executionMode == ExecutionMode.VERIFY_SAVEPOINT) {
nonParallelSource = new MigrationTestUtils.CheckingNonParallelSourceWithListState(NUM_SOURCE_ELEMENTS);
parallelSource = new MigrationTestUtils.CheckingParallelSourceWithUnionListState(NUM_SOURCE_ELEMENTS);
flatMap = new CheckingKeyedStateFlatMap();
timelyOperator = new CheckingTimelyStatefulOperator();
} else {
throw new IllegalStateException("Unknown ExecutionMode " + executionMode);
}
env
.addSource(nonParallelSource).uid("CheckpointingSource1")
.keyBy(0)
.flatMap(flatMap).startNewChain().uid("CheckpointingKeyedStateFlatMap1")
.keyBy(0)
.transform(
"timely_stateful_operator",
new TypeHint<Tuple2<Long, Long>>() {}.getTypeInfo(),
timelyOperator).uid("CheckpointingTimelyStatefulOperator1")
.addSink(new MigrationTestUtils.AccumulatorCountingSink<>());
env
.addSource(parallelSource).uid("CheckpointingSource2")
.keyBy(0)
.flatMap(flatMap).startNewChain().uid("CheckpointingKeyedStateFlatMap2")
.keyBy(0)
.transform(
"timely_stateful_operator",
new TypeHint<Tuple2<Long, Long>>() {}.getTypeInfo(),
timelyOperator).uid("CheckpointingTimelyStatefulOperator2")
.addSink(new MigrationTestUtils.AccumulatorCountingSink<>());
if (executionMode == ExecutionMode.PERFORM_SAVEPOINT) {
executeAndSavepoint(
env,
"src/test/resources/" + getSavepointPath(testMigrateVersion, testStateBackend),
new Tuple2<>(MigrationTestUtils.AccumulatorCountingSink.NUM_ELEMENTS_ACCUMULATOR, NUM_SOURCE_ELEMENTS * 2));
} else {
restoreAndExecute(
env,
getResourceFilename(getSavepointPath(testMigrateVersion, testStateBackend)),
new Tuple2<>(MigrationTestUtils.CheckingNonParallelSourceWithListState.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, 1),
new Tuple2<>(MigrationTestUtils.CheckingParallelSourceWithUnionListState.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, parallelism),
new Tuple2<>(CheckingKeyedStateFlatMap.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS * 2),
new Tuple2<>(CheckingTimelyStatefulOperator.SUCCESSFUL_PROCESS_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS * 2),
new Tuple2<>(CheckingTimelyStatefulOperator.SUCCESSFUL_EVENT_TIME_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS * 2),
new Tuple2<>(CheckingTimelyStatefulOperator.SUCCESSFUL_PROCESSING_TIME_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS * 2),
new Tuple2<>(MigrationTestUtils.AccumulatorCountingSink.NUM_ELEMENTS_ACCUMULATOR, NUM_SOURCE_ELEMENTS * 2));
}
}
Example #12
| Source File: KafkaConsumerTestBase.java From flink with Apache License 2.0 | 4 votes |
|
/**
* Test delete behavior and metrics for producer.
* @throws Exception
*/
public void runAllDeletesTest() throws Exception {
final String topic = "alldeletestest";
createTestTopic(topic, 1, 1);
final int elementCount = 300;
// ----------- Write some data into Kafka -------------------
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(1);
env.getConfig().setRestartStrategy(RestartStrategies.noRestart());
DataStream<Tuple2<byte[], PojoValue>> kvStream = env.addSource(new SourceFunction<Tuple2<byte[], PojoValue>>() {
@Override
public void run(SourceContext<Tuple2<byte[], PojoValue>> ctx) throws Exception {
Random rnd = new Random(1337);
for (long i = 0; i < elementCount; i++) {
final byte[] key = new byte[200];
rnd.nextBytes(key);
ctx.collect(new Tuple2<>(key, (PojoValue) null));
}
}
@Override
public void cancel() {
}
});
TypeInformationKeyValueSerializationSchema<byte[], PojoValue> schema = new TypeInformationKeyValueSerializationSchema<>(byte[].class, PojoValue.class, env.getConfig());
Properties producerProperties = FlinkKafkaProducerBase.getPropertiesFromBrokerList(brokerConnectionStrings);
producerProperties.setProperty("retries", "3");
producerProperties.putAll(secureProps);
kafkaServer.produceIntoKafka(kvStream, topic, schema, producerProperties, null);
env.execute("Write deletes to Kafka");
// ----------- Read the data again -------------------
env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(1);
env.getConfig().setRestartStrategy(RestartStrategies.noRestart());
Properties props = new Properties();
props.putAll(standardProps);
props.putAll(secureProps);
DataStream<Tuple2<byte[], PojoValue>> fromKafka = env.addSource(kafkaServer.getConsumer(topic, schema, props));
fromKafka.flatMap(new RichFlatMapFunction<Tuple2<byte[], PojoValue>, Object>() {
long counter = 0;
@Override
public void flatMap(Tuple2<byte[], PojoValue> value, Collector<Object> out) throws Exception {
// ensure that deleted messages are passed as nulls
assertNull(value.f1);
counter++;
if (counter == elementCount) {
// we got the right number of elements
throw new SuccessException();
}
}
});
tryExecute(env, "Read deletes from Kafka");
deleteTestTopic(topic);
}
Example #13
| Source File: KafkaConsumerTestBase.java From flink with Apache License 2.0 | 4 votes |
|
public void runKeyValueTest() throws Exception {
final String topic = "keyvaluetest";
createTestTopic(topic, 1, 1);
final int elementCount = 5000;
// ----------- Write some data into Kafka -------------------
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(1);
env.setRestartStrategy(RestartStrategies.noRestart());
DataStream<Tuple2<Long, PojoValue>> kvStream = env.addSource(new SourceFunction<Tuple2<Long, PojoValue>>() {
@Override
public void run(SourceContext<Tuple2<Long, PojoValue>> ctx) throws Exception {
Random rnd = new Random(1337);
for (long i = 0; i < elementCount; i++) {
PojoValue pojo = new PojoValue();
pojo.when = new Date(rnd.nextLong());
pojo.lon = rnd.nextLong();
pojo.lat = i;
// make every second key null to ensure proper "null" serialization
Long key = (i % 2 == 0) ? null : i;
ctx.collect(new Tuple2<>(key, pojo));
}
}
@Override
public void cancel() {
}
});
KeyedSerializationSchema<Tuple2<Long, PojoValue>> schema = new TypeInformationKeyValueSerializationSchema<>(Long.class, PojoValue.class, env.getConfig());
Properties producerProperties = FlinkKafkaProducerBase.getPropertiesFromBrokerList(brokerConnectionStrings);
producerProperties.setProperty("retries", "3");
kafkaServer.produceIntoKafka(kvStream, topic, schema, producerProperties, null);
env.execute("Write KV to Kafka");
// ----------- Read the data again -------------------
env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(1);
env.setRestartStrategy(RestartStrategies.noRestart());
KafkaDeserializationSchema<Tuple2<Long, PojoValue>> readSchema = new TypeInformationKeyValueSerializationSchema<>(Long.class, PojoValue.class, env.getConfig());
Properties props = new Properties();
props.putAll(standardProps);
props.putAll(secureProps);
DataStream<Tuple2<Long, PojoValue>> fromKafka = env.addSource(kafkaServer.getConsumer(topic, readSchema, props));
fromKafka.flatMap(new RichFlatMapFunction<Tuple2<Long, PojoValue>, Object>() {
long counter = 0;
@Override
public void flatMap(Tuple2<Long, PojoValue> value, Collector<Object> out) throws Exception {
// the elements should be in order.
Assert.assertTrue("Wrong value " + value.f1.lat, value.f1.lat == counter);
if (value.f1.lat % 2 == 0) {
assertNull("key was not null", value.f0);
} else {
Assert.assertTrue("Wrong value " + value.f0, value.f0 == counter);
}
counter++;
if (counter == elementCount) {
// we got the right number of elements
throw new SuccessException();
}
}
});
tryExecute(env, "Read KV from Kafka");
deleteTestTopic(topic);
}
Example #14
| Source File: FtrlPredictStreamOp.java From Alink with Apache License 2.0 | 4 votes |
|
@Override
public FtrlPredictStreamOp linkFrom(StreamOperator<?>... inputs) {
checkOpSize(2, inputs);
try {
DataStream<LinearModelData> modelstr = inputs[0].getDataStream()
.flatMap(new RichFlatMapFunction<Row, Tuple2<Integer, Row>>() {
@Override
public void flatMap(Row row, Collector<Tuple2<Integer, Row>> out)
throws Exception {
int numTasks = getRuntimeContext().getNumberOfParallelSubtasks();
for (int i = 0; i < numTasks; ++i) {
out.collect(Tuple2.of(i, row));
}
}
}).partitionCustom(new Partitioner<Integer>() {
@Override
public int partition(Integer key, int numPartitions) { return key; }
}, 0).map(new MapFunction<Tuple2<Integer, Row>, Row>() {
@Override
public Row map(Tuple2<Integer, Row> value) throws Exception {
return value.f1;
}
})
.flatMap(new CollectModel());
TypeInformation[] types = new TypeInformation[3];
String[] names = new String[3];
for (int i = 0; i < 3; ++i) {
names[i] = inputs[0].getSchema().getFieldNames()[i + 2];
types[i] = inputs[0].getSchema().getFieldTypes()[i + 2];
}
TableSchema modelSchema = new TableSchema(names, types);
/* predict samples */
DataStream<Row> prediction = inputs[1].getDataStream()
.connect(modelstr)
.flatMap(new PredictProcess(TableUtil.toSchemaJson(modelSchema),
TableUtil.toSchemaJson(inputs[1].getSchema()), this.getParams(), dataBridge));
this.setOutputTable(DataStreamConversionUtil.toTable(getMLEnvironmentId(), prediction,
new LinearModelMapper(modelSchema, inputs[1].getSchema(), getParams()).getOutputSchema()));
} catch (Exception ex) {
ex.printStackTrace();
throw new RuntimeException(ex.toString());
}
return this;
}
Example #15
| Source File: KafkaConsumerTestBase.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
public void runKeyValueTest() throws Exception {
final String topic = "keyvaluetest";
createTestTopic(topic, 1, 1);
final int elementCount = 5000;
// ----------- Write some data into Kafka -------------------
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(1);
env.setRestartStrategy(RestartStrategies.noRestart());
env.getConfig().disableSysoutLogging();
DataStream<Tuple2<Long, PojoValue>> kvStream = env.addSource(new SourceFunction<Tuple2<Long, PojoValue>>() {
@Override
public void run(SourceContext<Tuple2<Long, PojoValue>> ctx) throws Exception {
Random rnd = new Random(1337);
for (long i = 0; i < elementCount; i++) {
PojoValue pojo = new PojoValue();
pojo.when = new Date(rnd.nextLong());
pojo.lon = rnd.nextLong();
pojo.lat = i;
// make every second key null to ensure proper "null" serialization
Long key = (i % 2 == 0) ? null : i;
ctx.collect(new Tuple2<>(key, pojo));
}
}
@Override
public void cancel() {
}
});
KeyedSerializationSchema<Tuple2<Long, PojoValue>> schema = new TypeInformationKeyValueSerializationSchema<>(Long.class, PojoValue.class, env.getConfig());
Properties producerProperties = FlinkKafkaProducerBase.getPropertiesFromBrokerList(brokerConnectionStrings);
producerProperties.setProperty("retries", "3");
kafkaServer.produceIntoKafka(kvStream, topic, schema, producerProperties, null);
env.execute("Write KV to Kafka");
// ----------- Read the data again -------------------
env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(1);
env.setRestartStrategy(RestartStrategies.noRestart());
env.getConfig().disableSysoutLogging();
KafkaDeserializationSchema<Tuple2<Long, PojoValue>> readSchema = new TypeInformationKeyValueSerializationSchema<>(Long.class, PojoValue.class, env.getConfig());
Properties props = new Properties();
props.putAll(standardProps);
props.putAll(secureProps);
DataStream<Tuple2<Long, PojoValue>> fromKafka = env.addSource(kafkaServer.getConsumer(topic, readSchema, props));
fromKafka.flatMap(new RichFlatMapFunction<Tuple2<Long, PojoValue>, Object>() {
long counter = 0;
@Override
public void flatMap(Tuple2<Long, PojoValue> value, Collector<Object> out) throws Exception {
// the elements should be in order.
Assert.assertTrue("Wrong value " + value.f1.lat, value.f1.lat == counter);
if (value.f1.lat % 2 == 0) {
assertNull("key was not null", value.f0);
} else {
Assert.assertTrue("Wrong value " + value.f0, value.f0 == counter);
}
counter++;
if (counter == elementCount) {
// we got the right number of elements
throw new SuccessException();
}
}
});
tryExecute(env, "Read KV from Kafka");
deleteTestTopic(topic);
}
Example #16
| Source File: StatefulJobSavepointMigrationITCase.java From flink with Apache License 2.0 | 4 votes |
|
@Test
public void testSavepoint() throws Exception {
final int parallelism = 4;
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setRestartStrategy(RestartStrategies.noRestart());
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
switch (testStateBackend) {
case StateBackendLoader.ROCKSDB_STATE_BACKEND_NAME:
env.setStateBackend(new RocksDBStateBackend(new MemoryStateBackend()));
break;
case StateBackendLoader.MEMORY_STATE_BACKEND_NAME:
env.setStateBackend(new MemoryStateBackend());
break;
default:
throw new UnsupportedOperationException();
}
env.enableCheckpointing(500);
env.setParallelism(parallelism);
env.setMaxParallelism(parallelism);
SourceFunction<Tuple2<Long, Long>> nonParallelSource;
SourceFunction<Tuple2<Long, Long>> parallelSource;
RichFlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> flatMap;
OneInputStreamOperator<Tuple2<Long, Long>, Tuple2<Long, Long>> timelyOperator;
if (executionMode == ExecutionMode.PERFORM_SAVEPOINT) {
nonParallelSource = new MigrationTestUtils.CheckpointingNonParallelSourceWithListState(NUM_SOURCE_ELEMENTS);
parallelSource = new MigrationTestUtils.CheckpointingParallelSourceWithUnionListState(NUM_SOURCE_ELEMENTS);
flatMap = new CheckpointingKeyedStateFlatMap();
timelyOperator = new CheckpointingTimelyStatefulOperator();
} else if (executionMode == ExecutionMode.VERIFY_SAVEPOINT) {
nonParallelSource = new MigrationTestUtils.CheckingNonParallelSourceWithListState(NUM_SOURCE_ELEMENTS);
parallelSource = new MigrationTestUtils.CheckingParallelSourceWithUnionListState(NUM_SOURCE_ELEMENTS);
flatMap = new CheckingKeyedStateFlatMap();
timelyOperator = new CheckingTimelyStatefulOperator();
} else {
throw new IllegalStateException("Unknown ExecutionMode " + executionMode);
}
env
.addSource(nonParallelSource).uid("CheckpointingSource1")
.keyBy(0)
.flatMap(flatMap).startNewChain().uid("CheckpointingKeyedStateFlatMap1")
.keyBy(0)
.transform(
"timely_stateful_operator",
new TypeHint<Tuple2<Long, Long>>() {}.getTypeInfo(),
timelyOperator).uid("CheckpointingTimelyStatefulOperator1")
.addSink(new MigrationTestUtils.AccumulatorCountingSink<>());
env
.addSource(parallelSource).uid("CheckpointingSource2")
.keyBy(0)
.flatMap(flatMap).startNewChain().uid("CheckpointingKeyedStateFlatMap2")
.keyBy(0)
.transform(
"timely_stateful_operator",
new TypeHint<Tuple2<Long, Long>>() {}.getTypeInfo(),
timelyOperator).uid("CheckpointingTimelyStatefulOperator2")
.addSink(new MigrationTestUtils.AccumulatorCountingSink<>());
if (executionMode == ExecutionMode.PERFORM_SAVEPOINT) {
executeAndSavepoint(
env,
"src/test/resources/" + getSavepointPath(testMigrateVersion, testStateBackend),
new Tuple2<>(MigrationTestUtils.AccumulatorCountingSink.NUM_ELEMENTS_ACCUMULATOR, NUM_SOURCE_ELEMENTS * 2));
} else {
restoreAndExecute(
env,
getResourceFilename(getSavepointPath(testMigrateVersion, testStateBackend)),
new Tuple2<>(MigrationTestUtils.CheckingNonParallelSourceWithListState.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, 1),
new Tuple2<>(MigrationTestUtils.CheckingParallelSourceWithUnionListState.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, parallelism),
new Tuple2<>(CheckingKeyedStateFlatMap.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS * 2),
new Tuple2<>(CheckingTimelyStatefulOperator.SUCCESSFUL_PROCESS_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS * 2),
new Tuple2<>(CheckingTimelyStatefulOperator.SUCCESSFUL_EVENT_TIME_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS * 2),
new Tuple2<>(CheckingTimelyStatefulOperator.SUCCESSFUL_PROCESSING_TIME_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS * 2),
new Tuple2<>(MigrationTestUtils.AccumulatorCountingSink.NUM_ELEMENTS_ACCUMULATOR, NUM_SOURCE_ELEMENTS * 2));
}
}
Example #17
| Source File: KafkaConsumerTestBase.java From flink with Apache License 2.0 | 4 votes |
|
/**
* Test delete behavior and metrics for producer.
* @throws Exception
*/
public void runAllDeletesTest() throws Exception {
final String topic = "alldeletestest";
createTestTopic(topic, 1, 1);
final int elementCount = 300;
// ----------- Write some data into Kafka -------------------
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(1);
env.getConfig().setRestartStrategy(RestartStrategies.noRestart());
env.getConfig().disableSysoutLogging();
DataStream<Tuple2<byte[], PojoValue>> kvStream = env.addSource(new SourceFunction<Tuple2<byte[], PojoValue>>() {
@Override
public void run(SourceContext<Tuple2<byte[], PojoValue>> ctx) throws Exception {
Random rnd = new Random(1337);
for (long i = 0; i < elementCount; i++) {
final byte[] key = new byte[200];
rnd.nextBytes(key);
ctx.collect(new Tuple2<>(key, (PojoValue) null));
}
}
@Override
public void cancel() {
}
});
TypeInformationKeyValueSerializationSchema<byte[], PojoValue> schema = new TypeInformationKeyValueSerializationSchema<>(byte[].class, PojoValue.class, env.getConfig());
Properties producerProperties = FlinkKafkaProducerBase.getPropertiesFromBrokerList(brokerConnectionStrings);
producerProperties.setProperty("retries", "3");
producerProperties.putAll(secureProps);
kafkaServer.produceIntoKafka(kvStream, topic, schema, producerProperties, null);
env.execute("Write deletes to Kafka");
// ----------- Read the data again -------------------
env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(1);
env.getConfig().setRestartStrategy(RestartStrategies.noRestart());
env.getConfig().disableSysoutLogging();
Properties props = new Properties();
props.putAll(standardProps);
props.putAll(secureProps);
DataStream<Tuple2<byte[], PojoValue>> fromKafka = env.addSource(kafkaServer.getConsumer(topic, schema, props));
fromKafka.flatMap(new RichFlatMapFunction<Tuple2<byte[], PojoValue>, Object>() {
long counter = 0;
@Override
public void flatMap(Tuple2<byte[], PojoValue> value, Collector<Object> out) throws Exception {
// ensure that deleted messages are passed as nulls
assertNull(value.f1);
counter++;
if (counter == elementCount) {
// we got the right number of elements
throw new SuccessException();
}
}
});
tryExecute(env, "Read deletes from Kafka");
deleteTestTopic(topic);
}
Example #18
| Source File: KafkaConsumerTestBase.java From flink with Apache License 2.0 | 4 votes |
|
public void runKeyValueTest() throws Exception {
final String topic = "keyvaluetest";
createTestTopic(topic, 1, 1);
final int elementCount = 5000;
// ----------- Write some data into Kafka -------------------
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(1);
env.setRestartStrategy(RestartStrategies.noRestart());
env.getConfig().disableSysoutLogging();
DataStream<Tuple2<Long, PojoValue>> kvStream = env.addSource(new SourceFunction<Tuple2<Long, PojoValue>>() {
@Override
public void run(SourceContext<Tuple2<Long, PojoValue>> ctx) throws Exception {
Random rnd = new Random(1337);
for (long i = 0; i < elementCount; i++) {
PojoValue pojo = new PojoValue();
pojo.when = new Date(rnd.nextLong());
pojo.lon = rnd.nextLong();
pojo.lat = i;
// make every second key null to ensure proper "null" serialization
Long key = (i % 2 == 0) ? null : i;
ctx.collect(new Tuple2<>(key, pojo));
}
}
@Override
public void cancel() {
}
});
KeyedSerializationSchema<Tuple2<Long, PojoValue>> schema = new TypeInformationKeyValueSerializationSchema<>(Long.class, PojoValue.class, env.getConfig());
Properties producerProperties = FlinkKafkaProducerBase.getPropertiesFromBrokerList(brokerConnectionStrings);
producerProperties.setProperty("retries", "3");
kafkaServer.produceIntoKafka(kvStream, topic, schema, producerProperties, null);
env.execute("Write KV to Kafka");
// ----------- Read the data again -------------------
env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(1);
env.setRestartStrategy(RestartStrategies.noRestart());
env.getConfig().disableSysoutLogging();
KafkaDeserializationSchema<Tuple2<Long, PojoValue>> readSchema = new TypeInformationKeyValueSerializationSchema<>(Long.class, PojoValue.class, env.getConfig());
Properties props = new Properties();
props.putAll(standardProps);
props.putAll(secureProps);
DataStream<Tuple2<Long, PojoValue>> fromKafka = env.addSource(kafkaServer.getConsumer(topic, readSchema, props));
fromKafka.flatMap(new RichFlatMapFunction<Tuple2<Long, PojoValue>, Object>() {
long counter = 0;
@Override
public void flatMap(Tuple2<Long, PojoValue> value, Collector<Object> out) throws Exception {
// the elements should be in order.
Assert.assertTrue("Wrong value " + value.f1.lat, value.f1.lat == counter);
if (value.f1.lat % 2 == 0) {
assertNull("key was not null", value.f0);
} else {
Assert.assertTrue("Wrong value " + value.f0, value.f0 == counter);
}
counter++;
if (counter == elementCount) {
// we got the right number of elements
throw new SuccessException();
}
}
});
tryExecute(env, "Read KV from Kafka");
deleteTestTopic(topic);
}
Example #19
| Source File: TypeExtractorTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
@SuppressWarnings({ "unchecked", "rawtypes" })
@Test
public void testTupleWithTuples() {
// use getFlatMapReturnTypes()
RichFlatMapFunction<?, ?> function = new RichFlatMapFunction<Tuple3<Tuple1<String>, Tuple1<Integer>, Tuple2<Long, Long>>, Tuple3<Tuple1<String>, Tuple1<Integer>, Tuple2<Long, Long>>>() {
private static final long serialVersionUID = 1L;
@Override
public void flatMap(Tuple3<Tuple1<String>, Tuple1<Integer>, Tuple2<Long, Long>> value,
Collector<Tuple3<Tuple1<String>, Tuple1<Integer>, Tuple2<Long, Long>>> out) throws Exception {
// nothing to do
}
};
TypeInformation<?> ti = TypeExtractor.getFlatMapReturnTypes(function, (TypeInformation) TypeInformation.of(new TypeHint<Tuple3<Tuple1<String>, Tuple1<Integer>, Tuple2<Long, Long>>>(){}));
Assert.assertTrue(ti.isTupleType());
Assert.assertEquals(3, ti.getArity());
Assert.assertTrue(ti instanceof TupleTypeInfo);
List<FlatFieldDescriptor> ffd = new ArrayList<FlatFieldDescriptor>();
((TupleTypeInfo) ti).getFlatFields("f0.f0", 0, ffd);
Assert.assertEquals(0, ffd.get(0).getPosition() );
ffd.clear();
((TupleTypeInfo) ti).getFlatFields("f0.f0", 0, ffd);
Assert.assertTrue( ffd.get(0).getType() instanceof BasicTypeInfo );
Assert.assertTrue( ffd.get(0).getType().getTypeClass().equals(String.class) );
ffd.clear();
((TupleTypeInfo) ti).getFlatFields("f1.f0", 0, ffd);
Assert.assertEquals(1, ffd.get(0).getPosition() );
ffd.clear();
TupleTypeInfo<?> tti = (TupleTypeInfo<?>) ti;
Assert.assertEquals(Tuple3.class, tti.getTypeClass());
Assert.assertTrue(tti.getTypeAt(0).isTupleType());
Assert.assertTrue(tti.getTypeAt(1).isTupleType());
Assert.assertTrue(tti.getTypeAt(2).isTupleType());
Assert.assertEquals(Tuple1.class, tti.getTypeAt(0).getTypeClass());
Assert.assertEquals(Tuple1.class, tti.getTypeAt(1).getTypeClass());
Assert.assertEquals(Tuple2.class, tti.getTypeAt(2).getTypeClass());
Assert.assertEquals(1, tti.getTypeAt(0).getArity());
Assert.assertEquals(1, tti.getTypeAt(1).getArity());
Assert.assertEquals(2, tti.getTypeAt(2).getArity());
Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO, ((TupleTypeInfo<?>) tti.getTypeAt(0)).getTypeAt(0));
Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO, ((TupleTypeInfo<?>) tti.getTypeAt(1)).getTypeAt(0));
Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO, ((TupleTypeInfo<?>) tti.getTypeAt(2)).getTypeAt(0));
Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO, ((TupleTypeInfo<?>) tti.getTypeAt(2)).getTypeAt(1));
// use getForObject()
Tuple3<Tuple1<String>, Tuple1<Integer>, Tuple2<Long, Long>> t = new Tuple3<Tuple1<String>, Tuple1<Integer>, Tuple2<Long, Long>>(
new Tuple1<String>("hello"), new Tuple1<Integer>(1), new Tuple2<Long, Long>(2L, 3L));
Assert.assertTrue(TypeExtractor.getForObject(t) instanceof TupleTypeInfo);
TupleTypeInfo<?> tti2 = (TupleTypeInfo<?>) TypeExtractor.getForObject(t);
Assert.assertEquals(1, tti2.getTypeAt(0).getArity());
Assert.assertEquals(1, tti2.getTypeAt(1).getArity());
Assert.assertEquals(2, tti2.getTypeAt(2).getArity());
Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO, ((TupleTypeInfo<?>) tti2.getTypeAt(0)).getTypeAt(0));
Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO, ((TupleTypeInfo<?>) tti2.getTypeAt(1)).getTypeAt(0));
Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO, ((TupleTypeInfo<?>) tti2.getTypeAt(2)).getTypeAt(0));
Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO, ((TupleTypeInfo<?>) tti2.getTypeAt(2)).getTypeAt(1));
}
Example #20
| Source File: StatefulJobSavepointMigrationITCase.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
@Test
public void testSavepoint() throws Exception {
final int parallelism = 4;
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setRestartStrategy(RestartStrategies.noRestart());
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
switch (testStateBackend) {
case StateBackendLoader.ROCKSDB_STATE_BACKEND_NAME:
env.setStateBackend(new RocksDBStateBackend(new MemoryStateBackend()));
break;
case StateBackendLoader.MEMORY_STATE_BACKEND_NAME:
env.setStateBackend(new MemoryStateBackend());
break;
default:
throw new UnsupportedOperationException();
}
env.enableCheckpointing(500);
env.setParallelism(parallelism);
env.setMaxParallelism(parallelism);
SourceFunction<Tuple2<Long, Long>> nonParallelSource;
SourceFunction<Tuple2<Long, Long>> parallelSource;
RichFlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> flatMap;
OneInputStreamOperator<Tuple2<Long, Long>, Tuple2<Long, Long>> timelyOperator;
if (executionMode == ExecutionMode.PERFORM_SAVEPOINT) {
nonParallelSource = new MigrationTestUtils.CheckpointingNonParallelSourceWithListState(NUM_SOURCE_ELEMENTS);
parallelSource = new MigrationTestUtils.CheckpointingParallelSourceWithUnionListState(NUM_SOURCE_ELEMENTS);
flatMap = new CheckpointingKeyedStateFlatMap();
timelyOperator = new CheckpointingTimelyStatefulOperator();
} else if (executionMode == ExecutionMode.VERIFY_SAVEPOINT) {
nonParallelSource = new MigrationTestUtils.CheckingNonParallelSourceWithListState(NUM_SOURCE_ELEMENTS);
parallelSource = new MigrationTestUtils.CheckingParallelSourceWithUnionListState(NUM_SOURCE_ELEMENTS);
flatMap = new CheckingKeyedStateFlatMap();
timelyOperator = new CheckingTimelyStatefulOperator();
} else {
throw new IllegalStateException("Unknown ExecutionMode " + executionMode);
}
env
.addSource(nonParallelSource).uid("CheckpointingSource1")
.keyBy(0)
.flatMap(flatMap).startNewChain().uid("CheckpointingKeyedStateFlatMap1")
.keyBy(0)
.transform(
"timely_stateful_operator",
new TypeHint<Tuple2<Long, Long>>() {}.getTypeInfo(),
timelyOperator).uid("CheckpointingTimelyStatefulOperator1")
.addSink(new MigrationTestUtils.AccumulatorCountingSink<>());
env
.addSource(parallelSource).uid("CheckpointingSource2")
.keyBy(0)
.flatMap(flatMap).startNewChain().uid("CheckpointingKeyedStateFlatMap2")
.keyBy(0)
.transform(
"timely_stateful_operator",
new TypeHint<Tuple2<Long, Long>>() {}.getTypeInfo(),
timelyOperator).uid("CheckpointingTimelyStatefulOperator2")
.addSink(new MigrationTestUtils.AccumulatorCountingSink<>());
if (executionMode == ExecutionMode.PERFORM_SAVEPOINT) {
executeAndSavepoint(
env,
"src/test/resources/" + getSavepointPath(testMigrateVersion, testStateBackend),
new Tuple2<>(MigrationTestUtils.AccumulatorCountingSink.NUM_ELEMENTS_ACCUMULATOR, NUM_SOURCE_ELEMENTS * 2));
} else {
restoreAndExecute(
env,
getResourceFilename(getSavepointPath(testMigrateVersion, testStateBackend)),
new Tuple2<>(MigrationTestUtils.CheckingNonParallelSourceWithListState.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, 1),
new Tuple2<>(MigrationTestUtils.CheckingParallelSourceWithUnionListState.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, parallelism),
new Tuple2<>(CheckingKeyedStateFlatMap.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS * 2),
new Tuple2<>(CheckingTimelyStatefulOperator.SUCCESSFUL_PROCESS_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS * 2),
new Tuple2<>(CheckingTimelyStatefulOperator.SUCCESSFUL_EVENT_TIME_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS * 2),
new Tuple2<>(CheckingTimelyStatefulOperator.SUCCESSFUL_PROCESSING_TIME_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS * 2),
new Tuple2<>(MigrationTestUtils.AccumulatorCountingSink.NUM_ELEMENTS_ACCUMULATOR, NUM_SOURCE_ELEMENTS * 2));
}
}
Example #21
| Source File: KafkaConsumerTestBase.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
/**
* Test delete behavior and metrics for producer.
* @throws Exception
*/
public void runAllDeletesTest() throws Exception {
final String topic = "alldeletestest";
createTestTopic(topic, 1, 1);
final int elementCount = 300;
// ----------- Write some data into Kafka -------------------
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(1);
env.getConfig().setRestartStrategy(RestartStrategies.noRestart());
env.getConfig().disableSysoutLogging();
DataStream<Tuple2<byte[], PojoValue>> kvStream = env.addSource(new SourceFunction<Tuple2<byte[], PojoValue>>() {
@Override
public void run(SourceContext<Tuple2<byte[], PojoValue>> ctx) throws Exception {
Random rnd = new Random(1337);
for (long i = 0; i < elementCount; i++) {
final byte[] key = new byte[200];
rnd.nextBytes(key);
ctx.collect(new Tuple2<>(key, (PojoValue) null));
}
}
@Override
public void cancel() {
}
});
TypeInformationKeyValueSerializationSchema<byte[], PojoValue> schema = new TypeInformationKeyValueSerializationSchema<>(byte[].class, PojoValue.class, env.getConfig());
Properties producerProperties = FlinkKafkaProducerBase.getPropertiesFromBrokerList(brokerConnectionStrings);
producerProperties.setProperty("retries", "3");
producerProperties.putAll(secureProps);
kafkaServer.produceIntoKafka(kvStream, topic, schema, producerProperties, null);
env.execute("Write deletes to Kafka");
// ----------- Read the data again -------------------
env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(1);
env.getConfig().setRestartStrategy(RestartStrategies.noRestart());
env.getConfig().disableSysoutLogging();
Properties props = new Properties();
props.putAll(standardProps);
props.putAll(secureProps);
DataStream<Tuple2<byte[], PojoValue>> fromKafka = env.addSource(kafkaServer.getConsumer(topic, schema, props));
fromKafka.flatMap(new RichFlatMapFunction<Tuple2<byte[], PojoValue>, Object>() {
long counter = 0;
@Override
public void flatMap(Tuple2<byte[], PojoValue> value, Collector<Object> out) throws Exception {
// ensure that deleted messages are passed as nulls
assertNull(value.f1);
counter++;
if (counter == elementCount) {
// we got the right number of elements
throw new SuccessException();
}
}
});
tryExecute(env, "Read deletes from Kafka");
deleteTestTopic(topic);
}
