org.apache.flink.api.common.functions.RichFilterFunction Java Examples

The following examples show how to use org.apache.flink.api.common.functions.RichFilterFunction. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: AggregatorsITCase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testDistributedCacheWithIterations() throws Exception{
	final String testString = "Et tu, Brute?";
	final String testName = "testing_caesar";

	final File folder = tempFolder.newFolder();
	final File resultFile = new File(folder, UUID.randomUUID().toString());

	String testPath = resultFile.toString();
	String resultPath = resultFile.toURI().toString();

	File tempFile = new File(testPath);
	try (FileWriter writer = new FileWriter(tempFile)) {
		writer.write(testString);
	}

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.registerCachedFile(resultPath, testName);

	IterativeDataSet<Long> solution = env.fromElements(1L).iterate(2);
	solution.closeWith(env.generateSequence(1, 2).filter(new RichFilterFunction<Long>() {
		@Override
		public void open(Configuration parameters) throws Exception{
			File file = getRuntimeContext().getDistributedCache().getFile(testName);
			BufferedReader reader = new BufferedReader(new FileReader(file));
			String output = reader.readLine();
			reader.close();
			assertEquals(output, testString);
		}

		@Override
		public boolean filter(Long value) throws Exception {
			return false;
		}
	}).withBroadcastSet(solution, "SOLUTION")).output(new DiscardingOutputFormat<Long>());
	env.execute();
}
Example #2
Source File: AggregatorsITCase.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testDistributedCacheWithIterations() throws Exception{
	final String testString = "Et tu, Brute?";
	final String testName = "testing_caesar";

	final File folder = tempFolder.newFolder();
	final File resultFile = new File(folder, UUID.randomUUID().toString());

	String testPath = resultFile.toString();
	String resultPath = resultFile.toURI().toString();

	File tempFile = new File(testPath);
	try (FileWriter writer = new FileWriter(tempFile)) {
		writer.write(testString);
	}

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.registerCachedFile(resultPath, testName);

	IterativeDataSet<Long> solution = env.fromElements(1L).iterate(2);
	solution.closeWith(env.generateSequence(1, 2).filter(new RichFilterFunction<Long>() {
		@Override
		public void open(Configuration parameters) throws Exception{
			File file = getRuntimeContext().getDistributedCache().getFile(testName);
			BufferedReader reader = new BufferedReader(new FileReader(file));
			String output = reader.readLine();
			reader.close();
			assertEquals(output, testString);
		}

		@Override
		public boolean filter(Long value) throws Exception {
			return false;
		}
	}).withBroadcastSet(solution, "SOLUTION")).output(new DiscardingOutputFormat<Long>());
	env.execute();
}
Example #3
Source File: AggregatorsITCase.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testDistributedCacheWithIterations() throws Exception{
	final String testString = "Et tu, Brute?";
	final String testName = "testing_caesar";

	final File folder = tempFolder.newFolder();
	final File resultFile = new File(folder, UUID.randomUUID().toString());

	String testPath = resultFile.toString();
	String resultPath = resultFile.toURI().toString();

	File tempFile = new File(testPath);
	try (FileWriter writer = new FileWriter(tempFile)) {
		writer.write(testString);
	}

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.registerCachedFile(resultPath, testName);

	IterativeDataSet<Long> solution = env.fromElements(1L).iterate(2);
	solution.closeWith(env.generateSequence(1, 2).filter(new RichFilterFunction<Long>() {
		@Override
		public void open(Configuration parameters) throws Exception{
			File file = getRuntimeContext().getDistributedCache().getFile(testName);
			BufferedReader reader = new BufferedReader(new FileReader(file));
			String output = reader.readLine();
			reader.close();
			assertEquals(output, testString);
		}

		@Override
		public boolean filter(Long value) throws Exception {
			return false;
		}
	}).withBroadcastSet(solution, "SOLUTION")).output(new DiscardingOutputFormat<Long>());
	env.execute();
}
Example #4
Source File: GrepJobOptimized.java    From flink-perf with Apache License 2.0 4 votes vote down vote up 
public static void main(final String[] args) throws Exception {
	// set up the execution environment
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	String in = args[0];
	String out = args[1];
	System.err.println("Using input=" + in);
	System.err.println("Using output=" + out);

	String patterns[] = new String[args.length - 2];
	System.arraycopy(args, 2, patterns, 0, args.length - 2);
	System.err.println("Using patterns: " + Arrays.toString(patterns));

	// get input data
	DataSet<StringValue> text = env.createInput(new TextValueInputFormat(new Path(in)));

	for (int p = 0; p < patterns.length; p++) {
		final String pattern = patterns[p];
		DataSet<StringValue> res = text.filter(new RichFilterFunction<StringValue>() {
			Pattern p = Pattern.compile(pattern);

			@Override
			public void open(Configuration parameters) throws Exception {
				super.open(parameters);
			}

			@Override
			public boolean filter(StringValue valueIn) throws Exception {
				final String value = valueIn.getValue();
				if (value == null || value.length() == 0) {
					return false;
				}
				final Matcher m = p.matcher(value);
				if (m.find()) {
					return true;
				}
				return false;
			}
		}).name("grep for " + pattern);
		res.writeAsText(out + "_" + pattern, FileSystem.WriteMode.OVERWRITE);
	}

	// execute program
	JobExecutionResult jobResult = env.execute("Flink Grep benchmark");
	System.err.println(AccumulatorHelper.getResultsFormated(jobResult.getAllAccumulatorResults()));
}
Example #5
Source File: GrepJob.java    From flink-perf with Apache License 2.0 4 votes vote down vote up 
public static void main(final String[] args) throws Exception {
	// set up the execution environment
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	String in = args[0];
	String out = args[1];
	System.err.println("Using input=" + in);
	System.err.println("Using output=" + out);

	String patterns[] = new String[args.length - 2];
	System.arraycopy(args, 2, patterns, 0, args.length - 2);
	System.err.println("Using patterns: " + Arrays.toString(patterns));

	// get input data
	DataSet<String> text = env.readTextFile(args[0]);
	for (int p = 0; p < patterns.length; p++) {
		final String pattern = patterns[p];
		DataSet<String> res = text.filter(new RichFilterFunction<String>() {
			private static final long serialVersionUID = 1L;

			Pattern p = Pattern.compile(pattern);
			LongCounter filterMatches = new LongCounter();
			LongCounter filterRecords = new LongCounter();

			@Override
			public void open(Configuration parameters) throws Exception {
				super.open(parameters);
				getRuntimeContext().addAccumulator("filterMatchCount-" + pattern, filterMatches);
				getRuntimeContext().addAccumulator("filterRecordCount-" + pattern, filterRecords);
			}

			@Override
			public boolean filter(String value) throws Exception {
				filterRecords.add(1L);
				if (value == null || value.length() == 0) {
					return false;
				}
				final Matcher m = p.matcher(value);
				if (m.find()) {
					filterMatches.add(1L);
					return true;
				}
				return false;
			}
		}).name("grep for " + pattern);
		res.writeAsText(out + "_" + pattern, FileSystem.WriteMode.OVERWRITE);
	}

	// execute program
	JobExecutionResult jobResult = env.execute("Flink Grep benchmark");
	System.err.println(AccumulatorHelper.getResultsFormated(jobResult.getAllAccumulatorResults()));
}