org.apache.flink.api.common.functions.FilterFunction Java Examples

@SuppressWarnings("serial")
@Test
public void testFilterVertices() throws Exception {
	/*
	 * Test filterOnVertices:
	 */
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	Graph<Long, Long, Long> graph = Graph.fromDataSet(TestGraphUtils.getLongLongVertexData(env),
		TestGraphUtils.getLongLongEdgeData(env), env);

	DataSet<Edge<Long, Long>> data = graph.filterOnVertices(new FilterFunction<Vertex<Long, Long>>() {
		public boolean filter(Vertex<Long, Long> vertex) throws Exception {
			return (vertex.getValue() > 2);
		}
	}).getEdges();

	List<Edge<Long, Long>> result = data.collect();

	expectedResult = "3,4,34\n" +
		"3,5,35\n" +
		"4,5,45\n";

	compareResultAsTuples(result, expectedResult);
}
 

@SuppressWarnings("serial")
@Test
public void testFilterVertices() throws Exception {
	/*
	 * Test filterOnVertices:
	 */
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	Graph<Long, Long, Long> graph = Graph.fromDataSet(TestGraphUtils.getLongLongVertexData(env),
		TestGraphUtils.getLongLongEdgeData(env), env);

	DataSet<Edge<Long, Long>> data = graph.filterOnVertices(new FilterFunction<Vertex<Long, Long>>() {
		public boolean filter(Vertex<Long, Long> vertex) throws Exception {
			return (vertex.getValue() > 2);
		}
	}).getEdges();

	List<Edge<Long, Long>> result = data.collect();

	expectedResult = "3,4,34\n" +
		"3,5,35\n" +
		"4,5,45\n";

	compareResultAsTuples(result, expectedResult);
}
 

@SuppressWarnings("serial")
@Test
public void testFilterEdges() throws Exception {
	/*
	 * Test filterOnEdges:
	 */
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	Graph<Long, Long, Long> graph = Graph.fromDataSet(TestGraphUtils.getLongLongVertexData(env),
		TestGraphUtils.getLongLongEdgeData(env), env);

	DataSet<Edge<Long, Long>> data = graph.filterOnEdges(new FilterFunction<Edge<Long, Long>>() {
		public boolean filter(Edge<Long, Long> edge) throws Exception {
			return (edge.getValue() > 34);
		}
	}).getEdges();

	List<Edge<Long, Long>> result = data.collect();

	expectedResult = "3,5,35\n" +
		"4,5,45\n" +
		"5,1,51\n";

	compareResultAsTuples(result, expectedResult);
}
 

@Test
public void testDefaultName() {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<String> strs = env.fromCollection(Arrays.asList("a", "b"));

	// WARNING: The test will fail if this line is being moved down in the file (the line-number is hard-coded)
	strs.filter(new FilterFunction<String>() {
		private static final long serialVersionUID = 1L;

		@Override
		public boolean filter(String value) throws Exception {
			return value.equals("a");
		}
	}).output(new DiscardingOutputFormat<String>());
	Plan plan = env.createProgramPlan();
	testForName("Filter at testDefaultName(NamesTest.java:55)", plan);
}
 

@Override
public List <Row> read(FilterFunction <Row> filter) {
	if (filter == null) {
		return rows;
	}

	return rows.stream()
		.filter(x -> {
			try {
				return filter.filter(x);
			} catch (Exception e) {
				throw new RuntimeException(e);
			}
		})
		.collect(Collectors.toList());
}
 

public static void main(String[] args) throws Exception {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    env.createInput(new TableInputFormat<Tuple2<String, String>>() {

        private Tuple2<String, String> reuse = new Tuple2<String, String>();

        @Override
        protected Scan getScanner() {
            Scan scan = new Scan();
            scan.addColumn(INFO, BAR);
            return scan;
        }

        @Override
        protected String getTableName() {
            return HBASE_TABLE_NAME;
        }

        @Override
        protected Tuple2<String, String> mapResultToTuple(Result result) {
            String key = Bytes.toString(result.getRow());
            String val = Bytes.toString(result.getValue(INFO, BAR));
            reuse.setField(key, 0);
            reuse.setField(val, 1);
            return reuse;
        }
    }).filter(new FilterFunction<Tuple2<String, String>>() {
        @Override
        public boolean filter(Tuple2<String, String> value) throws Exception {
            return value.f1.startsWith("zhisheng");
        }
    }).print();
}
 

@Test
public void testInheritOverride() {
	// verify that we can explicitly disable inheritance of the input slot sharing groups

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	FilterFunction<Long> dummyFilter = new FilterFunction<Long>() {
		@Override
		public boolean filter(Long value) {
			return false;
		}
	};

	DataStream<Long> src1 = env.generateSequence(1, 10).slotSharingGroup("group-1");
	DataStream<Long> src2 = env.generateSequence(1, 10).slotSharingGroup("group-1");

	// this should not inherit group but be in "default"
	src1.union(src2).filter(dummyFilter).slotSharingGroup("default");
	JobGraph jobGraph = env.getStreamGraph().getJobGraph();

	List<JobVertex> vertices = jobGraph.getVerticesSortedTopologicallyFromSources();

	assertEquals(vertices.get(0).getSlotSharingGroup(), vertices.get(1).getSlotSharingGroup());
	assertNotEquals(vertices.get(0).getSlotSharingGroup(), vertices.get(2).getSlotSharingGroup());
	assertNotEquals(vertices.get(1).getSlotSharingGroup(), vertices.get(2).getSlotSharingGroup());
}
 

@SuppressWarnings("serial")
@Test
public void testFilterEdges() throws Exception {
	/*
	 * Test filterOnEdges:
	 */
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	Graph<Long, Long, Long> graph = Graph.fromDataSet(TestGraphUtils.getLongLongVertexData(env),
		TestGraphUtils.getLongLongEdgeData(env), env);

	DataSet<Edge<Long, Long>> data = graph.filterOnEdges(new FilterFunction<Edge<Long, Long>>() {
		public boolean filter(Edge<Long, Long> edge) throws Exception {
			return (edge.getValue() > 34);
		}
	}).getEdges();

	List<Edge<Long, Long>> result = data.collect();

	expectedResult = "3,5,35\n" +
		"4,5,45\n" +
		"5,1,51\n";

	compareResultAsTuples(result, expectedResult);
}
 

public RocksDBCheckpointIterator(IncrementalKeyedStateHandle handle, FilterFunction<String> stateFilter,
		String localPath) {
	this.localPath = localPath;
	this.cancelStreamRegistry = new CloseableRegistry();
	List<StateMetaInfoSnapshot> stateMetaInfoSnapshots = StateMetadataUtils
			.getKeyedBackendSerializationProxy(handle.getMetaStateHandle()).getStateMetaInfoSnapshots();

	stateColumnFamilyHandles = new ArrayList<>(stateMetaInfoSnapshots.size() + 1);
	List<ColumnFamilyDescriptor> stateColumnFamilyDescriptors = createAndRegisterColumnFamilyDescriptors(
			stateMetaInfoSnapshots);
	try {
		transferAllStateDataToDirectory(handle, new Path(localPath));
		this.db = openDB(localPath, stateColumnFamilyDescriptors, stateColumnFamilyHandles);
		createColumnIterators(stateFilter, stateMetaInfoSnapshots);
	} catch (Exception e) {
		throw new IllegalStateException(e);
	}
}
 

@Test
public void testDefaultName() {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<String> strs = env.fromCollection(Arrays.asList("a", "b"));

	// WARNING: The test will fail if this line is being moved down in the file (the line-number is hard-coded)
	strs.filter(new FilterFunction<String>() {
		private static final long serialVersionUID = 1L;

		@Override
		public boolean filter(String value) throws Exception {
			return value.equals("a");
		}
	}).output(new DiscardingOutputFormat<String>());
	Plan plan = env.createProgramPlan();
	testForName("Filter at testDefaultName(NamesTest.java:55)", plan);
}
 

public static void main(String[] args) throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

    //并行度为1
    env.setParallelism(1);

    //通过generateSequence得到Long类型的DataSource
    DataStream<Long> dataStream = env.generateSequence(1, 10);

    //做一次过滤，只保留偶数，然后打印
    dataStream.filter(new FilterFunction<Long>() {
        @Override
        public boolean filter(Long aLong) throws Exception {
            return 0L==aLong.longValue()%2L;
        }
    }).print();


    env.execute("API DataSource demo : collection");
}
 

@Test
public void testInheritOverride() {
	// verify that we can explicitly disable inheritance of the input slot sharing groups

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	FilterFunction<Long> dummyFilter = new FilterFunction<Long>() {
		@Override
		public boolean filter(Long value) {
			return false;
		}
	};

	DataStream<Long> src1 = env.generateSequence(1, 10).slotSharingGroup("group-1");
	DataStream<Long> src2 = env.generateSequence(1, 10).slotSharingGroup("group-1");

	// this should not inherit group but be in "default"
	src1.union(src2).filter(dummyFilter).slotSharingGroup("default");
	JobGraph jobGraph = env.getStreamGraph().getJobGraph();

	List<JobVertex> vertices = jobGraph.getVerticesSortedTopologicallyFromSources();

	assertEquals(vertices.get(0).getSlotSharingGroup(), vertices.get(1).getSlotSharingGroup());
	assertNotEquals(vertices.get(0).getSlotSharingGroup(), vertices.get(2).getSlotSharingGroup());
	assertNotEquals(vertices.get(1).getSlotSharingGroup(), vertices.get(2).getSlotSharingGroup());
}
 

/**
 * Function that verifies whether the edge to be removed is part of the SSSP or not.
 * If it is, the src vertex will be invalidated.
 *
 * @param edgeToBeRemoved
 * @param edgesInSSSP
 * @return true or false
 */
public static boolean isInSSSP(final Edge<Long, Double> edgeToBeRemoved, DataSet<Edge<Long, Double>> edgesInSSSP) throws Exception {

	return edgesInSSSP.filter(new FilterFunction<Edge<Long, Double>>() {
		@Override
		public boolean filter(Edge<Long, Double> edge) throws Exception {
			return edge.equals(edgeToBeRemoved);
		}
	}).count() > 0;
}
 

public TestOperator() {
	super(new FilterFunction<Object>() {
		@Override
		public boolean filter(Object value) {
			return false;
		}
	});
}
 

private DataSet<Tuple2<IntValue, IntValue>> getFilteredDataSet(ExecutionEnvironment env) {
	return getDataSet(env)
		.filter(new FilterFunction<Tuple2<IntValue, IntValue>>() {
			@Override
			public boolean filter(Tuple2<IntValue, IntValue> value) throws Exception {
				return (value.f0.getValue() % 2) == 0;
			}
		});
}
 

public static DataSet<DocCountVectorizerModelData> generateDocCountModel(Params params, BatchOperator in) {
    BatchOperator<?> docWordCnt = in.udtf(
            params.get(SELECTED_COL),
            new String[] {WORD_COL_NAME, DOC_WORD_COUNT_COL_NAME},
            new DocWordSplitCount(NLPConstant.WORD_DELIMITER),
            new String[] {});
    BatchOperator docCnt = in.select("COUNT(1) AS " + DOC_COUNT_COL_NAME);

    DataSet<Row> sortInput = docWordCnt
            .select(new String[] {WORD_COL_NAME, DOC_WORD_COUNT_COL_NAME})
            .getDataSet()
            .groupBy(0)
            .reduceGroup(new CalcIdf(params.get(MAX_DF), params.get(MIN_DF)))
            .withBroadcastSet(docCnt.getDataSet(), "docCnt");

    Tuple2<DataSet<Tuple2<Integer, Row>>, DataSet<Tuple2<Integer, Long>>> partitioned = SortUtils.pSort(
            sortInput, 1);
    DataSet<Tuple2<Long, Row>> ordered = localSort(partitioned.f0, partitioned.f1, 1);

    int vocabSize = params.get(VOCAB_SIZE);
    DataSet<DocCountVectorizerModelData> resDocCountModel = ordered.filter(new FilterFunction<Tuple2<Long, Row>>() {
        @Override
        public boolean filter(Tuple2<Long, Row> value) {
            return value.f0 < vocabSize;
        }
    }).mapPartition(new BuildDocCountModel(params)).setParallelism(1);
    return resDocCountModel;
}
 

public static FilterFunction<SamoaType> getFilter(final String streamID) {
	return new FilterFunction<SamoaType>() {
		@Override
		public boolean filter(SamoaType o) throws Exception {
			return o.f2.equals(streamID);
		}
	};
}
 

public static void main(String[] args) throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	DataStream<TemperatureEvent> inputEventStream = env.fromElements(new TemperatureEvent("xyz", 22.0),
			new TemperatureEvent("xyz", 20.1), new TemperatureEvent("xyz", 21.1), new TemperatureEvent("xyz", 22.2),
			new TemperatureEvent("xyz", 29.1), new TemperatureEvent("xyz", 22.3), new TemperatureEvent("xyz", 22.1),
			new TemperatureEvent("xyz", 22.4), new TemperatureEvent("xyz", 22.7),
			new TemperatureEvent("xyz", 27.0));

	Pattern<TemperatureEvent, ?> warningPattern = Pattern.<TemperatureEvent> begin("first")
			.subtype(TemperatureEvent.class).where(new FilterFunction<TemperatureEvent>() {
				private static final long serialVersionUID = 1L;

				public boolean filter(TemperatureEvent value) {
					if (value.getTemperature() >= 26.0) {
						return true;
					}
					return false;
				}
			}).within(Time.seconds(10));

	DataStream<Alert> patternStream = CEP.pattern(inputEventStream, warningPattern)
			.select(new PatternSelectFunction<TemperatureEvent, Alert>() {
				private static final long serialVersionUID = 1L;

				public Alert select(Map<String, TemperatureEvent> event) throws Exception {

					return new Alert("Temperature Rise Detected");
				}

			});

	patternStream.print();
	env.execute("CEP on Temperature Sensor");
}
 

@Test
public void testFilterModificationException2() {
	try {
		final UdfAnalyzer ua = new UdfAnalyzer(FilterFunction.class, FilterMod2.class, "operator",
			STRING_STRING_TUPLE2_TYPE_INFO, null, null, null, null, true);
		ua.analyze();
		Assert.fail();
	}
	catch (CodeErrorException e) {
		// ok
	}
}
 

@Test
public void testGivenName() {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<String> strs = env.fromCollection(Arrays.asList("a", "b"));
	strs.filter(new FilterFunction<String>() {
		private static final long serialVersionUID = 1L;
		@Override
		public boolean filter(String value) throws Exception {
			return value.equals("a");
		}
	}).name("GivenName").output(new DiscardingOutputFormat<String>());
	Plan plan = env.createProgramPlan();
	testForName("GivenName", plan);
}
 

@Test
public void testBulkIterationWithTerminationCriterion() {
	try {
		ExecutionEnvironment env = ExecutionEnvironment.createCollectionsEnvironment();

		IterativeDataSet<Integer> iteration = env.fromElements(1).iterate(100);

		DataSet<Integer> iterationResult = iteration.map(new AddSuperstepNumberMapper());

		DataSet<Integer> terminationCriterion = iterationResult.filter(new FilterFunction<Integer>() {
			public boolean filter(Integer value) {
				return value < 50;
			}
		});

		List<Integer> collected = new ArrayList<Integer>();

		iteration.closeWith(iterationResult, terminationCriterion)
				.output(new LocalCollectionOutputFormat<Integer>(collected));

		env.execute();

		assertEquals(1, collected.size());
		assertEquals(56, collected.get(0).intValue());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

public FilterOperator() {
	super(new FilterFunction<Object>() {
		@Override
		public boolean filter(Object value) {
			return false;
		}
	});
}
 

public static void main(String[] args) throws Exception {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    env.createInput(new TableInputFormat<Tuple2<String, String>>() {

        private Tuple2<String, String> reuse = new Tuple2<String, String>();

        @Override
        protected Scan getScanner() {
            Scan scan = new Scan();
            scan.addColumn(INFO, BAR);
            return scan;
        }

        @Override
        protected String getTableName() {
            return HBASE_TABLE_NAME;
        }

        @Override
        protected Tuple2<String, String> mapResultToTuple(Result result) {
            String key = Bytes.toString(result.getRow());
            String val = Bytes.toString(result.getValue(INFO, BAR));
            reuse.setField(key, 0);
            reuse.setField(val, 1);
            return reuse;
        }
    }).filter(new FilterFunction<Tuple2<String, String>>() {
        @Override
        public boolean filter(Tuple2<String, String> value) throws Exception {
            return value.f1.startsWith("zhisheng");
        }
    }).print();
}
 

/**
 * Apply filtering functions to the graph and return a sub-graph that
 * satisfies the predicates for both vertices and edges.
 *
 * @param vertexFilter the filter function for vertices.
 * @param edgeFilter the filter function for edges.
 * @return the resulting sub-graph.
 */
public Graph<K, VV, EV> subgraph(FilterFunction<Vertex<K, VV>> vertexFilter, FilterFunction<Edge<K, EV>> edgeFilter) {

	DataSet<Vertex<K, VV>> filteredVertices = this.vertices.filter(vertexFilter);

	DataSet<Edge<K, EV>> remainingEdges = this.edges.join(filteredVertices)
			.where(0).equalTo(0).with(new ProjectEdge<>())
			.join(filteredVertices).where(1).equalTo(0)
			.with(new ProjectEdge<>()).name("Subgraph");

	DataSet<Edge<K, EV>> filteredEdges = remainingEdges.filter(edgeFilter);

	return new Graph<>(filteredVertices, filteredEdges, this.context);
}
 

/**
 * Apply a filtering function to the graph and return a sub-graph that
 * satisfies the predicates only for the vertices.
 *
 * @param vertexFilter the filter function for vertices.
 * @return the resulting sub-graph.
 */
public Graph<K, VV, EV> filterOnVertices(FilterFunction<Vertex<K, VV>> vertexFilter) {

	DataSet<Vertex<K, VV>> filteredVertices = this.vertices.filter(vertexFilter);

	DataSet<Edge<K, EV>> remainingEdges = this.edges.join(filteredVertices)
			.where(0).equalTo(0).with(new ProjectEdge<>())
			.join(filteredVertices).where(1).equalTo(0)
			.with(new ProjectEdge<>()).name("Filter on vertices");

	return new Graph<>(filteredVertices, remainingEdges, this.context);
}
 

private void createColumnIterators(FilterFunction<String> stateFilter,
		List<StateMetaInfoSnapshot> stateMetaInfoSnapshots)
		throws Exception {
	Map<String, RocksIteratorWrapper> iterators = new HashMap<>();
	for (int i = 0; i < stateMetaInfoSnapshots.size(); i++) {
		String name = stateMetaInfoSnapshots.get(i).getName();
		if (stateFilter.filter(name)) {
			RocksIteratorWrapper iterator = new RocksIteratorWrapper(
					this.db.newIterator(stateColumnFamilyHandles.get(i + 1)));
			iterators.put(name, iterator);
			iterator.seekToFirst();
		}
	}

	iteratorQueue = new LinkedList<>(iterators.entrySet());
	updateCurrentIterator();
}
 

@SuppressWarnings("serial")
@Test
public void testSubGraph() throws Exception {
	/*
	 * Test subgraph:
	 */
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	Graph<Long, Long, Long> graph = Graph.fromDataSet(TestGraphUtils.getLongLongVertexData(env),
		TestGraphUtils.getLongLongEdgeData(env), env);

	DataSet<Edge<Long, Long>> data = graph.subgraph(
		new FilterFunction<Vertex<Long, Long>>() {
			public boolean filter(Vertex<Long, Long> vertex) throws Exception {
				return (vertex.getValue() > 2);
			}
		},
		new FilterFunction<Edge<Long, Long>>() {
			public boolean filter(Edge<Long, Long> edge) throws Exception {
				return (edge.getValue() > 34);
			}
		}).getEdges();

	List<Edge<Long, Long>> result = data.collect();

	expectedResult = "3,5,35\n" +
		"4,5,45\n";

	compareResultAsTuples(result, expectedResult);
}
 

public OperatorStateReader(ExecutionEnvironment env, Savepoint sp, String uid, Collection<String> stateNames) {
	this(env, sp, uid, new FilterFunction<String>() {
		private static final long serialVersionUID = 1L;
		HashSet<String> filtered = new HashSet<>(stateNames);

		@Override
		public boolean filter(String s) throws Exception {
			return filtered.contains(s);
		}
	});
}
 

@Test
public void testUnion() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	FilterFunction<Long> dummyFilter = new FilterFunction<Long>() {
		@Override
		public boolean filter(Long value) {
			return false;
		}
	};

	DataStream<Long> src1 = env.generateSequence(1, 10);
	DataStream<Long> src2 = env.generateSequence(1, 10).slotSharingGroup("src-1");

	// this should not inherit group "src-1"
	src1.union(src2).filter(dummyFilter);

	DataStream<Long> src3 = env.generateSequence(1, 10).slotSharingGroup("group-1");
	DataStream<Long> src4 = env.generateSequence(1, 10).slotSharingGroup("group-1");

	// this should inherit "group-1" now
	src3.union(src4).filter(dummyFilter);

	JobGraph jobGraph = env.getStreamGraph().getJobGraph();

	List<JobVertex> vertices = jobGraph.getVerticesSortedTopologicallyFromSources();

	// first pipeline
	assertEquals(vertices.get(0).getSlotSharingGroup(), vertices.get(4).getSlotSharingGroup());
	assertNotEquals(vertices.get(0).getSlotSharingGroup(), vertices.get(1).getSlotSharingGroup());
	assertNotEquals(vertices.get(1).getSlotSharingGroup(), vertices.get(4).getSlotSharingGroup());

	// second pipeline
	assertEquals(vertices.get(2).getSlotSharingGroup(), vertices.get(3).getSlotSharingGroup());
	assertEquals(vertices.get(2).getSlotSharingGroup(), vertices.get(5).getSlotSharingGroup());
	assertEquals(vertices.get(3).getSlotSharingGroup(), vertices.get(5).getSlotSharingGroup());
}
 

public TestOperator() {
	super(new FilterFunction<Object>() {
		@Override
		public boolean filter(Object value) {
			return false;
		}
	});
}