org.apache.flink.streaming.api.operators.StreamFlatMap Java Examples

The following examples show how to use org.apache.flink.streaming.api.operators.StreamFlatMap. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: KeyedStateInputFormatTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testReadState() throws Exception {
	OperatorID operatorID = OperatorIDGenerator.fromUid("uid");

	OperatorSubtaskState state = createOperatorSubtaskState(new StreamFlatMap<>(new StatefulFunction()));
	OperatorState operatorState = new OperatorState(operatorID, 1, 128);
	operatorState.putState(0, state);

	KeyedStateInputFormat<?, ?> format = new KeyedStateInputFormat<>(operatorState, new MemoryStateBackend(), Types.INT, new ReaderFunction());
	KeyGroupRangeInputSplit split = format.createInputSplits(1)[0];

	KeyedStateReaderFunction<Integer, Integer> userFunction = new ReaderFunction();

	List<Integer> data = readInputSplit(split, userFunction);

	Assert.assertEquals("Incorrect data read from input split", Arrays.asList(1, 2, 3), data);
}
Example #2
Source File: KeyedStateInputFormatTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testReadMultipleOutputPerKey() throws Exception {
	OperatorID operatorID = OperatorIDGenerator.fromUid("uid");

	OperatorSubtaskState state = createOperatorSubtaskState(new StreamFlatMap<>(new StatefulFunction()));
	OperatorState operatorState = new OperatorState(operatorID, 1, 128);
	operatorState.putState(0, state);

	KeyedStateInputFormat<?, ?> format = new KeyedStateInputFormat<>(operatorState, new MemoryStateBackend(), Types.INT, new ReaderFunction());
	KeyGroupRangeInputSplit split = format.createInputSplits(1)[0];

	KeyedStateReaderFunction<Integer, Integer> userFunction = new DoubleReaderFunction();

	List<Integer> data = readInputSplit(split, userFunction);

	Assert.assertEquals("Incorrect data read from input split", Arrays.asList(1, 1, 2, 2, 3, 3), data);
}
Example #3
Source File: KeyedStateInputFormatTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test(expected = IOException.class)
public void testInvalidProcessReaderFunctionFails() throws Exception {
	OperatorID operatorID = OperatorIDGenerator.fromUid("uid");

	OperatorSubtaskState state = createOperatorSubtaskState(new StreamFlatMap<>(new StatefulFunction()));
	OperatorState operatorState = new OperatorState(operatorID, 1, 128);
	operatorState.putState(0, state);

	KeyedStateInputFormat<?, ?> format = new KeyedStateInputFormat<>(operatorState, new MemoryStateBackend(), Types.INT, new ReaderFunction());
	KeyGroupRangeInputSplit split = format.createInputSplits(1)[0];

	KeyedStateReaderFunction<Integer, Integer> userFunction = new InvalidReaderFunction();

	readInputSplit(split, userFunction);

	Assert.fail("KeyedStateReaderFunction did not fail on invalid RuntimeContext use");
}
Example #4
Source File: KeyedStateInputFormatTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testReadState() throws Exception {
	OperatorID operatorID = OperatorIDGenerator.fromUid("uid");

	OperatorSubtaskState state = createOperatorSubtaskState(new StreamFlatMap<>(new StatefulFunction()));
	OperatorState operatorState = new OperatorState(operatorID, 1, 128);
	operatorState.putState(0, state);

	KeyedStateInputFormat<?, ?, ?> format = new KeyedStateInputFormat<>(operatorState, new MemoryStateBackend(), new Configuration(), new KeyedStateReaderOperator<>(new ReaderFunction(), Types.INT));
	KeyGroupRangeInputSplit split = format.createInputSplits(1)[0];

	KeyedStateReaderFunction<Integer, Integer> userFunction = new ReaderFunction();

	List<Integer> data = readInputSplit(split, userFunction);

	Assert.assertEquals("Incorrect data read from input split", Arrays.asList(1, 2, 3), data);
}
Example #5
Source File: KeyedStateInputFormatTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testReadMultipleOutputPerKey() throws Exception {
	OperatorID operatorID = OperatorIDGenerator.fromUid("uid");

	OperatorSubtaskState state = createOperatorSubtaskState(new StreamFlatMap<>(new StatefulFunction()));
	OperatorState operatorState = new OperatorState(operatorID, 1, 128);
	operatorState.putState(0, state);

	KeyedStateInputFormat<?, ?, ?> format = new KeyedStateInputFormat<>(operatorState, new MemoryStateBackend(), new Configuration(), new KeyedStateReaderOperator<>(new ReaderFunction(), Types.INT));
	KeyGroupRangeInputSplit split = format.createInputSplits(1)[0];

	KeyedStateReaderFunction<Integer, Integer> userFunction = new DoubleReaderFunction();

	List<Integer> data = readInputSplit(split, userFunction);

	Assert.assertEquals("Incorrect data read from input split", Arrays.asList(1, 1, 2, 2, 3, 3), data);
}
Example #6
Source File: KeyedStateInputFormatTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test(expected = IOException.class)
public void testInvalidProcessReaderFunctionFails() throws Exception {
	OperatorID operatorID = OperatorIDGenerator.fromUid("uid");

	OperatorSubtaskState state = createOperatorSubtaskState(new StreamFlatMap<>(new StatefulFunction()));
	OperatorState operatorState = new OperatorState(operatorID, 1, 128);
	operatorState.putState(0, state);

	KeyedStateInputFormat<?, ?, ?> format = new KeyedStateInputFormat<>(operatorState, new MemoryStateBackend(), new Configuration(), new KeyedStateReaderOperator<>(new ReaderFunction(), Types.INT));
	KeyGroupRangeInputSplit split = format.createInputSplits(1)[0];

	KeyedStateReaderFunction<Integer, Integer> userFunction = new InvalidReaderFunction();

	readInputSplit(split, userFunction);

	Assert.fail("KeyedStateReaderFunction did not fail on invalid RuntimeContext use");
}
Example #7
Source File: KeyedStateInputFormatTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testCreatePartitionedInputSplits() throws Exception {
	OperatorID operatorID = OperatorIDGenerator.fromUid("uid");

	OperatorSubtaskState state = createOperatorSubtaskState(new StreamFlatMap<>(new StatefulFunction()));
	OperatorState operatorState = new OperatorState(operatorID, 1, 128);
	operatorState.putState(0, state);

	KeyedStateInputFormat<?, ?> format = new KeyedStateInputFormat<>(operatorState, new MemoryStateBackend(), Types.INT, new ReaderFunction());
	KeyGroupRangeInputSplit[] splits = format.createInputSplits(4);
	Assert.assertEquals("Failed to properly partition operator state into input splits", 4, splits.length);
}
Example #8
Source File: KeyedStateInputFormatTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testMaxParallelismRespected() throws Exception {
	OperatorID operatorID = OperatorIDGenerator.fromUid("uid");

	OperatorSubtaskState state = createOperatorSubtaskState(new StreamFlatMap<>(new StatefulFunction()));
	OperatorState operatorState = new OperatorState(operatorID, 1, 128);
	operatorState.putState(0, state);

	KeyedStateInputFormat<?, ?> format = new KeyedStateInputFormat<>(operatorState, new MemoryStateBackend(), Types.INT, new ReaderFunction());
	KeyGroupRangeInputSplit[] splits = format.createInputSplits(129);
	Assert.assertEquals("Failed to properly partition operator state into input splits", 128, splits.length);
}
Example #9
Source File: KeyedStateInputFormatTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testCreatePartitionedInputSplits() throws Exception {
	OperatorID operatorID = OperatorIDGenerator.fromUid("uid");

	OperatorSubtaskState state = createOperatorSubtaskState(new StreamFlatMap<>(new StatefulFunction()));
	OperatorState operatorState = new OperatorState(operatorID, 1, 128);
	operatorState.putState(0, state);

	KeyedStateInputFormat<?, ?, ?> format = new KeyedStateInputFormat<>(operatorState, new MemoryStateBackend(), new Configuration(), new KeyedStateReaderOperator<>(new ReaderFunction(), Types.INT));
	KeyGroupRangeInputSplit[] splits = format.createInputSplits(4);
	Assert.assertEquals("Failed to properly partition operator state into input splits", 4, splits.length);
}
Example #10
Source File: KeyedStateInputFormatTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testMaxParallelismRespected() throws Exception {
	OperatorID operatorID = OperatorIDGenerator.fromUid("uid");

	OperatorSubtaskState state = createOperatorSubtaskState(new StreamFlatMap<>(new StatefulFunction()));
	OperatorState operatorState = new OperatorState(operatorID, 1, 128);
	operatorState.putState(0, state);

	KeyedStateInputFormat<?, ?, ?> format = new KeyedStateInputFormat<>(operatorState, new MemoryStateBackend(), new Configuration(), new KeyedStateReaderOperator<>(new ReaderFunction(), Types.INT));
	KeyGroupRangeInputSplit[] splits = format.createInputSplits(129);
	Assert.assertEquals("Failed to properly partition operator state into input splits", 128, splits.length);
}
Example #11
Source File: ListStateInputFormatTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
private OneInputStreamOperatorTestHarness<Integer, Void> getTestHarness() throws Exception {
	return new OneInputStreamOperatorTestHarness<>(new StreamFlatMap<>(new StatefulFunction()), IntSerializer.INSTANCE);
}
Example #12
Source File: UnionStateInputFormatTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
private OneInputStreamOperatorTestHarness<Integer, Void> getTestHarness() throws Exception {
	return new OneInputStreamOperatorTestHarness<>(new StreamFlatMap<>(new StatefulFunction()), IntSerializer.INSTANCE);
}
Example #13
Source File: UnionStateInputFormatTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
private OneInputStreamOperatorTestHarness<Integer, Void> getTestHarness() throws Exception {
	return new OneInputStreamOperatorTestHarness<>(new StreamFlatMap<>(new StatefulFunction()), IntSerializer.INSTANCE);
}
Example #14
Source File: ListStateInputFormatTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
private OneInputStreamOperatorTestHarness<Integer, Void> getTestHarness() throws Exception {
	return new OneInputStreamOperatorTestHarness<>(new StreamFlatMap<>(new StatefulFunction()), IntSerializer.INSTANCE);
}
Example #15
Source File: DataStream.java    From flink with Apache License 2.0 2 votes vote down vote up 
/**
 * Applies a FlatMap transformation on a {@link DataStream}. The
 * transformation calls a {@link FlatMapFunction} for each element of the
 * DataStream. Each FlatMapFunction call can return any number of elements
 * including none. The user can also extend {@link RichFlatMapFunction} to
 * gain access to other features provided by the
 * {@link org.apache.flink.api.common.functions.RichFunction} interface.
 *
 * @param flatMapper
 *            The FlatMapFunction that is called for each element of the
 *            DataStream
 * @param outputType {@link TypeInformation} for the result type of the function.
 *
 * @param <R>
 *            output type
 * @return The transformed {@link DataStream}.
 */
public <R> SingleOutputStreamOperator<R> flatMap(FlatMapFunction<T, R> flatMapper, TypeInformation<R> outputType) {
	return transform("Flat Map", outputType, new StreamFlatMap<>(clean(flatMapper)));

}
Example #16
Source File: DataStream.java    From flink with Apache License 2.0 1 votes vote down vote up 
/**
 * Applies a FlatMap transformation on a {@link DataStream}. The
 * transformation calls a {@link FlatMapFunction} for each element of the
 * DataStream. Each FlatMapFunction call can return any number of elements
 * including none. The user can also extend {@link RichFlatMapFunction} to
 * gain access to other features provided by the
 * {@link org.apache.flink.api.common.functions.RichFunction} interface.
 *
 * @param flatMapper
 *            The FlatMapFunction that is called for each element of the
 *            DataStream
 *
 * @param <R>
 *            output type
 * @return The transformed {@link DataStream}.
 */
public <R> SingleOutputStreamOperator<R> flatMap(FlatMapFunction<T, R> flatMapper) {

	TypeInformation<R> outType = TypeExtractor.getFlatMapReturnTypes(clean(flatMapper),
			getType(), Utils.getCallLocationName(), true);

	return transform("Flat Map", outType, new StreamFlatMap<>(clean(flatMapper)));

}
Example #17
Source File: DataStream.java    From Flink-CEPplus with Apache License 2.0 1 votes vote down vote up 
/**
 * Applies a FlatMap transformation on a {@link DataStream}. The
 * transformation calls a {@link FlatMapFunction} for each element of the
 * DataStream. Each FlatMapFunction call can return any number of elements
 * including none. The user can also extend {@link RichFlatMapFunction} to
 * gain access to other features provided by the
 * {@link org.apache.flink.api.common.functions.RichFunction} interface.
 *
 * @param flatMapper
 *            The FlatMapFunction that is called for each element of the
 *            DataStream
 *
 * @param <R>
 *            output type
 * @return The transformed {@link DataStream}.
 */
public <R> SingleOutputStreamOperator<R> flatMap(FlatMapFunction<T, R> flatMapper) {

	TypeInformation<R> outType = TypeExtractor.getFlatMapReturnTypes(clean(flatMapper),
			getType(), Utils.getCallLocationName(), true);

	return transform("Flat Map", outType, new StreamFlatMap<>(clean(flatMapper)));

}