org.apache.flink.streaming.api.datastream.SplitStream Java Examples

The following examples show how to use org.apache.flink.streaming.api.datastream.SplitStream. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: DirectedOutputITCase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void outputSelectorTest() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(3);

	TestListResultSink<Long> evenSink = new TestListResultSink<Long>();
	TestListResultSink<Long> oddAndTenSink = new TestListResultSink<Long>();
	TestListResultSink<Long> evenAndOddSink = new TestListResultSink<Long>();
	TestListResultSink<Long> allSink = new TestListResultSink<Long>();

	SplitStream<Long> source = env.generateSequence(1, 11).split(new MyOutputSelector());
	source.select(EVEN).addSink(evenSink);
	source.select(ODD, TEN).addSink(oddAndTenSink);
	source.select(EVEN, ODD).addSink(evenAndOddSink);
	source.addSink(allSink);

	env.execute();
	assertEquals(Arrays.asList(2L, 4L, 6L, 8L, 10L), evenSink.getSortedResult());
	assertEquals(Arrays.asList(1L, 3L, 5L, 7L, 9L, 10L, 11L), oddAndTenSink.getSortedResult());
	assertEquals(Arrays.asList(1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 10L, 11L),
			evenAndOddSink.getSortedResult());
	assertEquals(Arrays.asList(1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 10L, 11L),
			allSink.getSortedResult());
}
Example #2
Source File: DirectedOutputITCase.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void outputSelectorTest() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(3);

	TestListResultSink<Long> evenSink = new TestListResultSink<Long>();
	TestListResultSink<Long> oddAndTenSink = new TestListResultSink<Long>();
	TestListResultSink<Long> evenAndOddSink = new TestListResultSink<Long>();
	TestListResultSink<Long> allSink = new TestListResultSink<Long>();

	SplitStream<Long> source = env.generateSequence(1, 11).split(new MyOutputSelector());
	source.select(EVEN).addSink(evenSink);
	source.select(ODD, TEN).addSink(oddAndTenSink);
	source.select(EVEN, ODD).addSink(evenAndOddSink);
	source.addSink(allSink);

	env.execute();
	assertEquals(Arrays.asList(2L, 4L, 6L, 8L, 10L), evenSink.getSortedResult());
	assertEquals(Arrays.asList(1L, 3L, 5L, 7L, 9L, 10L, 11L), oddAndTenSink.getSortedResult());
	assertEquals(Arrays.asList(1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 10L, 11L),
			evenAndOddSink.getSortedResult());
	assertEquals(Arrays.asList(1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 10L, 11L),
			allSink.getSortedResult());
}
Example #3
Source File: JavaDataStreamTransformationApp.java    From 163-bigdate-note with GNU General Public License v3.0 6 votes vote down vote up 
private static void splitFunction(StreamExecutionEnvironment environment) {
    DataStreamSource<Long> dataStreamSource = environment.addSource(new JavaCustomNonParallelSourceFunction());

    SplitStream<Long> splitStream = dataStreamSource.split(new OutputSelector<Long>() {
        @Override
        public Iterable<String> select(Long value) {
            List<String> list = new ArrayList<>();
            if (value % 2 == 0) {
                list.add("event");
            } else {
                list.add("odd");
            }
            return list;
        }
    });

    splitStream.select("odd").print().setParallelism(1);
}
Example #4
Source File: DirectedOutputITCase.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void outputSelectorTest() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(3);

	TestListResultSink<Long> evenSink = new TestListResultSink<Long>();
	TestListResultSink<Long> oddAndTenSink = new TestListResultSink<Long>();
	TestListResultSink<Long> evenAndOddSink = new TestListResultSink<Long>();
	TestListResultSink<Long> allSink = new TestListResultSink<Long>();

	SplitStream<Long> source = env.generateSequence(1, 11).split(new MyOutputSelector());
	source.select(EVEN).addSink(evenSink);
	source.select(ODD, TEN).addSink(oddAndTenSink);
	source.select(EVEN, ODD).addSink(evenAndOddSink);
	source.addSink(allSink);

	env.execute();
	assertEquals(Arrays.asList(2L, 4L, 6L, 8L, 10L), evenSink.getSortedResult());
	assertEquals(Arrays.asList(1L, 3L, 5L, 7L, 9L, 10L, 11L), oddAndTenSink.getSortedResult());
	assertEquals(Arrays.asList(1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 10L, 11L),
			evenAndOddSink.getSortedResult());
	assertEquals(Arrays.asList(1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 10L, 11L),
			allSink.getSortedResult());
}
Example #5
Source File: SplitEvent.java    From flink-learning with Apache License 2.0 5 votes vote down vote up 
public static void main(String[] args) throws Exception {
    final ParameterTool params = ParameterTool.fromArgs(args);
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.getConfig().setGlobalJobParameters(params);

    DataStreamSource<MetricEvent> data = KafkaConfigUtil.buildSource(env);  //从 Kafka 获取到所有的数据流
    SplitStream<MetricEvent> splitData = data.split(new OutputSelector<MetricEvent>() {
        @Override
        public Iterable<String> select(MetricEvent metricEvent) {
            List<String> tags = new ArrayList<>();
            String type = metricEvent.getTags().get("type");
            switch (type) {
                case "machine":
                    tags.add("machine");
                    break;
                case "docker":
                    tags.add("docker");
                    break;
                case "application":
                    tags.add("application");
                    break;
                case "middleware":
                    tags.add("middleware");
                    break;
                default:
                    break;
            }
            return tags;
        }
    });

    DataStream<MetricEvent> machine = splitData.select("machine");
    DataStream<MetricEvent> docker = splitData.select("docker");
    DataStream<MetricEvent> application = splitData.select("application");
    DataStream<MetricEvent> middleware = splitData.select("middleware");

}
Example #6
Source File: SplitEvent.java    From flink-learning with Apache License 2.0 5 votes vote down vote up 
public static void main(String[] args) throws Exception {
    final ParameterTool params = ParameterTool.fromArgs(args);
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.getConfig().setGlobalJobParameters(params);

    DataStreamSource<MetricEvent> data = KafkaConfigUtil.buildSource(env);  //从 Kafka 获取到所有的数据流
    SplitStream<MetricEvent> splitData = data.split(new OutputSelector<MetricEvent>() {
        @Override
        public Iterable<String> select(MetricEvent metricEvent) {
            List<String> tags = new ArrayList<>();
            String type = metricEvent.getTags().get("type");
            switch (type) {
                case "machine":
                    tags.add("machine");
                    break;
                case "docker":
                    tags.add("docker");
                    break;
                case "application":
                    tags.add("application");
                    break;
                case "middleware":
                    tags.add("middleware");
                    break;
                default:
                    break;
            }
            return tags;
        }
    });

    DataStream<MetricEvent> machine = splitData.select("machine");
    DataStream<MetricEvent> docker = splitData.select("docker");
    DataStream<MetricEvent> application = splitData.select("application");
    DataStream<MetricEvent> middleware = splitData.select("middleware");

}
Example #7
Source File: PythonSplitStream.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
PythonSplitStream(SplitStream<PyObject> splitStream) {
	super(splitStream);
}
Example #8
Source File: IterateExample.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
public static void main(String[] args) throws Exception {

		// Checking input parameters
		final ParameterTool params = ParameterTool.fromArgs(args);

		// set up input for the stream of integer pairs

		// obtain execution environment and set setBufferTimeout to 1 to enable
		// continuous flushing of the output buffers (lowest latency)
		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment()
				.setBufferTimeout(1);

		// make parameters available in the web interface
		env.getConfig().setGlobalJobParameters(params);

		// create input stream of integer pairs
		DataStream<Tuple2<Integer, Integer>> inputStream;
		if (params.has("input")) {
			inputStream = env.readTextFile(params.get("input")).map(new FibonacciInputMap());
		} else {
			System.out.println("Executing Iterate example with default input data set.");
			System.out.println("Use --input to specify file input.");
			inputStream = env.addSource(new RandomFibonacciSource());
		}

		// create an iterative data stream from the input with 5 second timeout
		IterativeStream<Tuple5<Integer, Integer, Integer, Integer, Integer>> it = inputStream.map(new InputMap())
				.iterate(5000);

		// apply the step function to get the next Fibonacci number
		// increment the counter and split the output with the output selector
		SplitStream<Tuple5<Integer, Integer, Integer, Integer, Integer>> step = it.map(new Step())
				.split(new MySelector());

		// close the iteration by selecting the tuples that were directed to the
		// 'iterate' channel in the output selector
		it.closeWith(step.select("iterate"));

		// to produce the final output select the tuples directed to the
		// 'output' channel then get the input pairs that have the greatest iteration counter
		// on a 1 second sliding window
		DataStream<Tuple2<Tuple2<Integer, Integer>, Integer>> numbers = step.select("output")
				.map(new OutputMap());

		// emit results
		if (params.has("output")) {
			numbers.writeAsText(params.get("output"));
		} else {
			System.out.println("Printing result to stdout. Use --output to specify output path.");
			numbers.print();
		}

		// execute the program
		env.execute("Streaming Iteration Example");
	}
Example #9
Source File: IterateExample.java    From flink-learning with Apache License 2.0 4 votes vote down vote up 
public static void main(String[] args) throws Exception {

        final ParameterTool params = ParameterTool.fromArgs(args);
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment().setBufferTimeout(1);
        env.getConfig().setGlobalJobParameters(params);



        IterativeStream<Tuple5<Integer, Integer, Integer, Integer, Integer>> it = env.addSource(new RandomFibonacciSource())
                .map(new InputMap())
                .iterate(5000);

        SplitStream<Tuple5<Integer, Integer, Integer, Integer, Integer>> step = it.map(new Step())
                .split(new MySelector());

        it.closeWith(step.select("iterate"));

        step.select("output")
                .map(new OutputMap())
                .print();

        env.execute("Streaming Iteration Example");
    }
Example #10
Source File: IterateExample.java    From flink with Apache License 2.0 4 votes vote down vote up 
public static void main(String[] args) throws Exception {

		// Checking input parameters
		final ParameterTool params = ParameterTool.fromArgs(args);

		// set up input for the stream of integer pairs

		// obtain execution environment and set setBufferTimeout to 1 to enable
		// continuous flushing of the output buffers (lowest latency)
		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment()
				.setBufferTimeout(1);

		// make parameters available in the web interface
		env.getConfig().setGlobalJobParameters(params);

		// create input stream of integer pairs
		DataStream<Tuple2<Integer, Integer>> inputStream;
		if (params.has("input")) {
			inputStream = env.readTextFile(params.get("input")).map(new FibonacciInputMap());
		} else {
			System.out.println("Executing Iterate example with default input data set.");
			System.out.println("Use --input to specify file input.");
			inputStream = env.addSource(new RandomFibonacciSource());
		}

		// create an iterative data stream from the input with 5 second timeout
		IterativeStream<Tuple5<Integer, Integer, Integer, Integer, Integer>> it = inputStream.map(new InputMap())
				.iterate(5000);

		// apply the step function to get the next Fibonacci number
		// increment the counter and split the output with the output selector
		SplitStream<Tuple5<Integer, Integer, Integer, Integer, Integer>> step = it.map(new Step())
				.split(new MySelector());

		// close the iteration by selecting the tuples that were directed to the
		// 'iterate' channel in the output selector
		it.closeWith(step.select("iterate"));

		// to produce the final output select the tuples directed to the
		// 'output' channel then get the input pairs that have the greatest iteration counter
		// on a 1 second sliding window
		DataStream<Tuple2<Tuple2<Integer, Integer>, Integer>> numbers = step.select("output")
				.map(new OutputMap());

		// emit results
		if (params.has("output")) {
			numbers.writeAsText(params.get("output"));
		} else {
			System.out.println("Printing result to stdout. Use --output to specify output path.");
			numbers.print();
		}

		// execute the program
		env.execute("Streaming Iteration Example");
	}
Example #11
Source File: IterateExample.java    From flink-learning with Apache License 2.0 4 votes vote down vote up 
public static void main(String[] args) throws Exception {

        final ParameterTool params = ParameterTool.fromArgs(args);
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment().setBufferTimeout(1);
        env.getConfig().setGlobalJobParameters(params);



        IterativeStream<Tuple5<Integer, Integer, Integer, Integer, Integer>> it = env.addSource(new RandomFibonacciSource())
                .map(new InputMap())
                .iterate(5000);

        SplitStream<Tuple5<Integer, Integer, Integer, Integer, Integer>> step = it.map(new Step())
                .split(new MySelector());

        it.closeWith(step.select("iterate"));

        step.select("output")
                .map(new OutputMap())
                .print();

        env.execute("Streaming Iteration Example");
    }
Example #12
Source File: IterateExample.java    From flink with Apache License 2.0 4 votes vote down vote up 
public static void main(String[] args) throws Exception {

		// Checking input parameters
		final ParameterTool params = ParameterTool.fromArgs(args);

		// set up input for the stream of integer pairs

		// obtain execution environment and set setBufferTimeout to 1 to enable
		// continuous flushing of the output buffers (lowest latency)
		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment()
				.setBufferTimeout(1);

		// make parameters available in the web interface
		env.getConfig().setGlobalJobParameters(params);

		// create input stream of integer pairs
		DataStream<Tuple2<Integer, Integer>> inputStream;
		if (params.has("input")) {
			inputStream = env.readTextFile(params.get("input")).map(new FibonacciInputMap());
		} else {
			System.out.println("Executing Iterate example with default input data set.");
			System.out.println("Use --input to specify file input.");
			inputStream = env.addSource(new RandomFibonacciSource());
		}

		// create an iterative data stream from the input with 5 second timeout
		IterativeStream<Tuple5<Integer, Integer, Integer, Integer, Integer>> it = inputStream.map(new InputMap())
				.iterate(5000L);

		// apply the step function to get the next Fibonacci number
		// increment the counter and split the output with the output selector
		SplitStream<Tuple5<Integer, Integer, Integer, Integer, Integer>> step = it.map(new Step())
				.split(new MySelector());

		// close the iteration by selecting the tuples that were directed to the
		// 'iterate' channel in the output selector
		it.closeWith(step.select("iterate"));

		// to produce the final output select the tuples directed to the
		// 'output' channel then get the input pairs that have the greatest iteration counter
		// on a 1 second sliding window
		DataStream<Tuple2<Tuple2<Integer, Integer>, Integer>> numbers = step.select("output")
				.map(new OutputMap());

		// emit results
		if (params.has("output")) {
			numbers.writeAsText(params.get("output"));
		} else {
			System.out.println("Printing result to stdout. Use --output to specify output path.");
			numbers.print();
		}

		// execute the program
		env.execute("Streaming Iteration Example");
	}