Java Code Examples for org.apache.flink.table.api.java.StreamTableEnvironment#registerFunction()
The following examples show how to use
org.apache.flink.table.api.java.StreamTableEnvironment#registerFunction() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: ExecutionContext.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
|
private void registerTemporalTable(TemporalTableEntry temporalTableEntry) {
try {
final Table table = tableEnv.scan(temporalTableEntry.getHistoryTable());
final TableFunction<?> function = table.createTemporalTableFunction(
temporalTableEntry.getTimeAttribute(),
String.join(",", temporalTableEntry.getPrimaryKeyFields()));
if (tableEnv instanceof StreamTableEnvironment) {
StreamTableEnvironment streamTableEnvironment = (StreamTableEnvironment) tableEnv;
streamTableEnvironment.registerFunction(temporalTableEntry.getName(), function);
} else {
BatchTableEnvironment batchTableEnvironment = (BatchTableEnvironment) tableEnv;
batchTableEnvironment.registerFunction(temporalTableEntry.getName(), function);
}
} catch (Exception e) {
throw new SqlExecutionException(
"Invalid temporal table '" + temporalTableEntry.getName() + "' over table '" +
temporalTableEntry.getHistoryTable() + ".\nCause: " + e.getMessage());
}
}
Example 2
| Source File: ExecutionContext.java From flink with Apache License 2.0 | 6 votes |
|
private void registerTemporalTable(TemporalTableEntry temporalTableEntry) {
try {
final Table table = tableEnv.scan(temporalTableEntry.getHistoryTable());
final TableFunction<?> function = table.createTemporalTableFunction(
temporalTableEntry.getTimeAttribute(),
String.join(",", temporalTableEntry.getPrimaryKeyFields()));
if (tableEnv instanceof StreamTableEnvironment) {
StreamTableEnvironment streamTableEnvironment = (StreamTableEnvironment) tableEnv;
streamTableEnvironment.registerFunction(temporalTableEntry.getName(), function);
} else {
BatchTableEnvironment batchTableEnvironment = (BatchTableEnvironment) tableEnv;
batchTableEnvironment.registerFunction(temporalTableEntry.getName(), function);
}
} catch (Exception e) {
throw new SqlExecutionException(
"Invalid temporal table '" + temporalTableEntry.getName() + "' over table '" +
temporalTableEntry.getHistoryTable() + ".\nCause: " + e.getMessage());
}
}
Example 3
| Source File: UDTFStreamOp.java From Alink with Apache License 2.0 | 6 votes |
|
@Override
public UDTFStreamOp linkFrom(StreamOperator<?>... inputs) {
if (null == getFunc() && null == getFuncName()) {
throw new IllegalArgumentException("A TableFunction or a registered function name must be set using setFunc or setFuncName.");
}
StreamOperator<?> in = checkAndGetFirst(inputs);
String[] reservedCols = ObjectUtils.defaultIfNull(getReservedCols(), in.getColNames());
StreamTableEnvironment tEnv = MLEnvironmentFactory.get(getMLEnvironmentId()).getStreamTableEnvironment();
String funcName = getFuncName();
if (null == funcName) {
funcName = UDFHelper.generateRandomFuncName();
tEnv.registerFunction(funcName, func);
}
String clause = UDFHelper.generateUDTFClause(in.getOutputTable().toString(), funcName,
getOutputCols(), getSelectedCols(), reservedCols);
this.setOutputTable(tEnv.sqlQuery(clause));
return this;
}
Example 4
| Source File: UDFStreamOp.java From Alink with Apache License 2.0 | 6 votes |
|
@Override
public UDFStreamOp linkFrom(StreamOperator<?>... inputs) {
if (null == getFunc() && null == getFuncName()) {
throw new IllegalArgumentException("A ScalarFunction or a registered function name must be set using setFunc or setFuncName.");
}
StreamOperator<?> in = checkAndGetFirst(inputs);
String[] reservedCols = ObjectUtils.defaultIfNull(getReservedCols(), in.getColNames());
StreamTableEnvironment tEnv = MLEnvironmentFactory.get(getMLEnvironmentId()).getStreamTableEnvironment();
String funcName = getFuncName();
if (null == funcName) {
funcName = UDFHelper.generateRandomFuncName();
tEnv.registerFunction(funcName, func);
}
String clause = UDFHelper.generateUDFClause(in.getOutputTable().toString(), funcName,
getOutputCol(), getSelectedCols(), reservedCols);
setOutputTable(tEnv.sqlQuery(clause));
return this;
}
Example 5
| Source File: HBaseConnectorITCase.java From flink with Apache License 2.0 | 5 votes |
|
@Test
public void testHBaseLookupFunction() throws Exception {
StreamExecutionEnvironment streamEnv = StreamExecutionEnvironment.getExecutionEnvironment();
StreamTableEnvironment streamTableEnv = StreamTableEnvironment.create(streamEnv, streamSettings);
StreamITCase.clear();
// prepare a source table
DataStream<Row> ds = streamEnv.fromCollection(testData2).returns(testTypeInfo2);
Table in = streamTableEnv.fromDataStream(ds, "a, b, c");
streamTableEnv.registerTable("src", in);
Map<String, String> tableProperties = hbaseTableProperties();
TableSource source = TableFactoryService
.find(HBaseTableFactory.class, tableProperties)
.createTableSource(tableProperties);
streamTableEnv.registerFunction("hbaseLookup", ((HBaseTableSource) source).getLookupFunction(new String[]{ROWKEY}));
// perform a temporal table join query
String sqlQuery = "SELECT a,family1.col1, family3.col3 FROM src, LATERAL TABLE(hbaseLookup(a))";
Table result = streamTableEnv.sqlQuery(sqlQuery);
DataStream<Row> resultSet = streamTableEnv.toAppendStream(result, Row.class);
resultSet.addSink(new StreamITCase.StringSink<>());
streamEnv.execute();
List<String> expected = new ArrayList<>();
expected.add("1,10,Welt-1");
expected.add("2,20,Welt-2");
expected.add("3,30,Welt-3");
expected.add("3,30,Welt-3");
StreamITCase.compareWithList(expected);
}
Example 6
| Source File: AbstractFlinkClient.java From alchemy with Apache License 2.0 | 5 votes |
|
private void register(StreamTableEnvironment env, String name, Object function) {
if (function instanceof TableFunction) {
env.registerFunction(name, (TableFunction) function);
} else if (function instanceof AggregateFunction) {
env.registerFunction(name, (AggregateFunction) function);
} else if (function instanceof ScalarFunction) {
env.registerFunction(name, (ScalarFunction) function);
} else {
throw new RuntimeException("Unknown UDF {} was found." + name);
}
LOGGER.info("register udf, name:{}, class:{}", name, function.getClass());
}
Example 7
| Source File: JDBCLookupFunctionITCase.java From flink with Apache License 2.0 | 4 votes |
|
@Test
public void test() throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
StreamTableEnvironment tEnv = StreamTableEnvironment.create(env);
StreamITCase.clear();
Table t = tEnv.fromDataStream(env.fromCollection(Arrays.asList(
new Tuple2<>(1, 1),
new Tuple2<>(1, 1),
new Tuple2<>(2, 3),
new Tuple2<>(2, 5),
new Tuple2<>(3, 5),
new Tuple2<>(3, 8)
)), "id1, id2");
tEnv.registerTable("T", t);
JDBCTableSource.Builder builder = JDBCTableSource.builder()
.setOptions(JDBCOptions.builder()
.setDBUrl(DB_URL)
.setTableName(LOOKUP_TABLE)
.build())
.setSchema(TableSchema.builder().fields(
new String[]{"id1", "id2", "comment1", "comment2"},
new DataType[]{DataTypes.INT(), DataTypes.INT(), DataTypes.STRING(), DataTypes.STRING()})
.build());
if (useCache) {
builder.setLookupOptions(JDBCLookupOptions.builder()
.setCacheMaxSize(1000).setCacheExpireMs(1000 * 1000).build());
}
tEnv.registerFunction("jdbcLookup",
builder.build().getLookupFunction(t.getSchema().getFieldNames()));
String sqlQuery = "SELECT id1, id2, comment1, comment2 FROM T, " +
"LATERAL TABLE(jdbcLookup(id1, id2)) AS S(l_id1, l_id2, comment1, comment2)";
Table result = tEnv.sqlQuery(sqlQuery);
DataStream<Row> resultSet = tEnv.toAppendStream(result, Row.class);
resultSet.addSink(new StreamITCase.StringSink<>());
env.execute();
List<String> expected = new ArrayList<>();
expected.add("1,1,11-c1-v1,11-c2-v1");
expected.add("1,1,11-c1-v1,11-c2-v1");
expected.add("1,1,11-c1-v2,11-c2-v2");
expected.add("1,1,11-c1-v2,11-c2-v2");
expected.add("2,3,null,23-c2");
expected.add("2,5,25-c1,25-c2");
expected.add("3,8,38-c1,38-c2");
StreamITCase.compareWithList(expected);
}
Example 8
| Source File: AreasTotalPerHourOfDay.java From infoworld-post with Apache License 2.0 | 4 votes |
|
public static void main(String[] args) throws Exception {
// check parameter
if (args.length != 1) {
System.err.println("Please provide the path to the taxi rides file as a parameter");
}
String inputPath = args[0];
// create execution environment
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
// configure event-time and watermarks
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
env.getConfig().setAutoWatermarkInterval(1000L);
// create table environment
StreamTableEnvironment tEnv = TableEnvironment.getTableEnvironment(env);
// register user-defined function
tEnv.registerFunction("toCellId", new GeoUtils.ToCellId());
// get taxi ride event stream
DataStream<TaxiRide> rides = TaxiRides.getRides(env, inputPath);
// register taxi ride event stream as table "Rides"
tEnv.registerDataStream(
"Rides",
rides,
"medallion, licenseId, pickUpTime, dropOffTime.rowtime, " +
"pickUpLon, pickUpLat, dropOffLon, dropOffLat, total");
// define SQL query to compute average total per area and hour of day.
Table result = tEnv.sqlQuery(
"SELECT " +
" toCellId(dropOffLon, dropOffLat) AS area, " +
" EXTRACT(HOUR FROM dropOffTime) AS hourOfDay, " +
" AVG(total) AS avgTotal " +
"FROM Rides " +
"GROUP BY " +
" toCellId(dropOffLon, dropOffLat), " +
" EXTRACT(HOUR FROM dropOffTime)"
);
// convert result table into a retraction stream and print it
tEnv.toRetractStream(result, Row.class)
.print();
// execute the query
env.execute();
}
Example 9
| Source File: AreasTotalPerHour.java From infoworld-post with Apache License 2.0 | 4 votes |
|
public static void main(String[] args) throws Exception {
// check parameter
if (args.length != 1) {
System.err.println("Please provide the path to the taxi rides file as a parameter");
}
String inputPath = args[0];
// create execution environment
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
// configure event-time and watermarks
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
env.getConfig().setAutoWatermarkInterval(1000L);
// create table environment
StreamTableEnvironment tEnv = TableEnvironment.getTableEnvironment(env);
// register user-defined function
tEnv.registerFunction("toCellId", new GeoUtils.ToCellId());
// get taxi ride event stream
DataStream<TaxiRide> rides = TaxiRides.getRides(env, inputPath);
// register taxi ride event stream as table "Rides"
tEnv.registerDataStream(
"Rides",
rides,
"medallion, licenseId, pickUpTime, dropOffTime.rowtime, " +
"pickUpLon, pickUpLat, dropOffLon, dropOffLat, total");
// define SQL query to compute average total per area and hour
Table result = tEnv.sqlQuery(
"SELECT " +
" toCellId(dropOffLon, dropOffLat) AS area, " +
" TUMBLE_START(dropOffTime, INTERVAL '1' HOUR) AS t, " +
" AVG(total) AS avgTotal " +
"FROM Rides " +
"GROUP BY " +
" toCellId(dropOffLon, dropOffLat), " +
" TUMBLE(dropOffTime, INTERVAL '1' HOUR)"
);
// convert result table into an append stream and print it
tEnv.toAppendStream(result, Row.class)
.print();
// execute the query
env.execute();
}
Example 10
| Source File: TestTableFunction.java From yauaa with Apache License 2.0 | 4 votes |
|
@Test
public void testFunctionExtractDirect() throws Exception {
// The base input stream
StreamExecutionEnvironment senv = StreamExecutionEnvironment.getExecutionEnvironment();
DataStreamSource<Tuple3<String, String, String>> inputStream = getTestAgentStream(senv);
// The table environment
StreamTableEnvironment tableEnv = StreamTableEnvironment.create(senv);
// Give the stream a Table Name
tableEnv.createTemporaryView("AgentStream", inputStream, "useragent, expectedDeviceClass, expectedAgentNameVersionMajor");
// register the function
tableEnv.registerFunction("ParseUserAgent", new AnalyzeUseragentFunction("DeviceClass", "AgentNameVersionMajor"));
// The downside of doing it this way is that the parsing function (i.e. parsing and converting all results into a map)
// is called for each field you want. So in this simple case twice.
String sqlQuery =
"SELECT useragent,"+
" ParseUserAgent(useragent)['DeviceClass' ] as DeviceClass," +
" ParseUserAgent(useragent)['AgentNameVersionMajor'] as AgentNameVersionMajor," +
" expectedDeviceClass," +
" expectedAgentNameVersionMajor " +
"FROM AgentStream";
Table resultTable = tableEnv.sqlQuery(sqlQuery);
TypeInformation<Row> tupleType = new RowTypeInfo(STRING, STRING, STRING, STRING, STRING);
DataStream<Row> resultSet = tableEnv.toAppendStream(resultTable, tupleType);
resultSet.map((MapFunction<Row, String>) row -> {
Object useragent = row.getField(0);
Object deviceClass = row.getField(1);
Object agentNameVersionMajor = row.getField(2);
Object expectedDeviceClass = row.getField(3);
Object expectedAgentNameVersionMajor = row.getField(4);
assertTrue(useragent instanceof String);
assertTrue(deviceClass instanceof String);
assertTrue(agentNameVersionMajor instanceof String);
assertTrue(expectedDeviceClass instanceof String);
assertTrue(expectedAgentNameVersionMajor instanceof String);
assertEquals(expectedDeviceClass, deviceClass, "Wrong DeviceClass: " + useragent);
assertEquals(expectedAgentNameVersionMajor, agentNameVersionMajor, "Wrong AgentNameVersionMajor: " + useragent);
return useragent.toString();
}).printToErr();
senv.execute();
}
Example 11
| Source File: TestTableFunction.java From yauaa with Apache License 2.0 | 4 votes |
|
@Test
public void testMapFunctionExtractInSQLSubSelect() throws Exception {
// The base input stream
StreamExecutionEnvironment senv = StreamExecutionEnvironment.getExecutionEnvironment();
DataStreamSource<Tuple3<String, String, String>> inputStream = getTestAgentStream(senv);
// The table environment
StreamTableEnvironment tableEnv = StreamTableEnvironment.create(senv);
// Give the stream a Table Name
tableEnv.createTemporaryView("AgentStream", inputStream, "useragent, expectedDeviceClass, expectedAgentNameVersionMajor");
// register the function
tableEnv.registerFunction("ParseUserAgent", new AnalyzeUseragentFunction("DeviceClass", "AgentNameVersionMajor"));
String sqlQuery =
"SELECT useragent,"+
" parsedUseragent['DeviceClass'] AS deviceClass," +
" parsedUseragent['AgentNameVersionMajor'] AS agentNameVersionMajor," +
" expectedDeviceClass," +
" expectedAgentNameVersionMajor " +
"FROM ( " +
" SELECT useragent," +
" ParseUserAgent(useragent) AS parsedUseragent," +
" expectedDeviceClass," +
" expectedAgentNameVersionMajor " +
" FROM AgentStream " +
")";
Table resultTable = tableEnv.sqlQuery(sqlQuery);
TypeInformation<Row> tupleType = new RowTypeInfo(STRING, STRING, STRING, STRING, STRING);
DataStream<Row> resultSet = tableEnv.toAppendStream(resultTable, tupleType);
resultSet.map((MapFunction<Row, String>) row -> {
Object useragent = row.getField(0);
Object deviceClass = row.getField(1);
Object agentNameVersionMajor = row.getField(2);
Object expectedDeviceClass = row.getField(3);
Object expectedAgentNameVersionMajor = row.getField(4);
assertTrue(useragent instanceof String);
assertTrue(deviceClass instanceof String);
assertTrue(agentNameVersionMajor instanceof String);
assertTrue(expectedDeviceClass instanceof String);
assertTrue(expectedAgentNameVersionMajor instanceof String);
assertEquals(expectedDeviceClass, deviceClass, "Wrong DeviceClass: " + useragent);
assertEquals(expectedAgentNameVersionMajor, agentNameVersionMajor, "Wrong AgentNameVersionMajor: " + useragent);
return useragent.toString();
}).printToErr();
senv.execute();
}
Example 12
| Source File: TestTableFunction.java From yauaa with Apache License 2.0 | 4 votes |
|
@Test
public void testMapFunctionReturnMap() throws Exception {
// The base input stream
StreamExecutionEnvironment senv = StreamExecutionEnvironment.getExecutionEnvironment();
DataStreamSource<Tuple3<String, String, String>> inputStream = getTestAgentStream(senv);
// The table environment
StreamTableEnvironment tableEnv = StreamTableEnvironment.create(senv);
// Give the stream a Table Name
tableEnv.createTemporaryView("AgentStream", inputStream, "useragent, expectedDeviceClass, expectedAgentNameVersionMajor");
// register the function
tableEnv.registerFunction("ParseUserAgent", new AnalyzeUseragentFunction("DeviceClass", "AgentNameVersionMajor"));
String sqlQuery =
"SELECT useragent," +
" ParseUserAgent(useragent) AS parsedUseragent," +
" expectedDeviceClass AS expectedDeviceClass," +
" expectedAgentNameVersionMajor AS expectedAgentNameVersionMajor " +
"FROM AgentStream";
Table resultTable = tableEnv.sqlQuery(sqlQuery);
TypeInformation<Row> tupleType = new RowTypeInfo(STRING, MAP(STRING, STRING), STRING, STRING);
DataStream<Row> resultSet = tableEnv.toAppendStream(resultTable, tupleType);
resultSet.map((MapFunction<Row, String>) row -> {
Object useragent = row.getField(0);
Object parsedUseragent = row.getField(1);
Object expectedDeviceClass = row.getField(2);
Object expectedAgentNameVersionMajor = row.getField(3);
assertTrue(useragent instanceof String);
assertTrue(parsedUseragent instanceof Map<?, ?>);
assertTrue(expectedDeviceClass instanceof String);
assertTrue(expectedAgentNameVersionMajor instanceof String);
assertEquals(
expectedDeviceClass,
((Map<?, ?>)parsedUseragent).get("DeviceClass"),
"Wrong DeviceClass: " + useragent);
assertEquals(
expectedAgentNameVersionMajor,
((Map<?, ?>)parsedUseragent).get("AgentNameVersionMajor"),
"Wrong AgentNameVersionMajor: " + useragent);
return useragent.toString();
}).printToErr();
senv.execute();
}
Example 13
| Source File: DemonstrationOfTumblingTableSQLFunction.java From yauaa with Apache License 2.0 | 4 votes |
|
@Disabled
@Test
public void runDemonstration() throws Exception {
// The base input stream
StreamExecutionEnvironment senv = StreamExecutionEnvironment.getExecutionEnvironment();
senv.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
senv.getConfig().setAutoWatermarkInterval(1000);
DataStream<Tuple4<Long, String, String, String>> inputStream = senv
.addSource(new UAStreamSource())
.assignTimestampsAndWatermarks(new UAWatermarker());
// The table environment
StreamTableEnvironment tableEnv = StreamTableEnvironment.create(senv);
// Give the stream a Table Name
tableEnv.createTemporaryView("AgentStream", inputStream, "eventTime.rowtime, useragent, expectedDeviceClass, expectedAgentNameVersionMajor");
// register the function
tableEnv.registerFunction("ParseUserAgent", new AnalyzeUseragentFunction("DeviceClass", "AgentNameVersionMajor"));
int windowIntervalCount = 5;
String windowIntervalScale = "MINUTE";
String sqlQuery = String.format(
"SELECT" +
" TUMBLE_START(eventTime, INTERVAL '%d' %s) AS wStart," +
" deviceClass," +
" agentNameVersionMajor," +
" expectedDeviceClass," +
" expectedAgentNameVersionMajor," +
" Count('') " +
"FROM ( "+
" SELECT " +
" eventTime, " +
" parsedUserAgent['DeviceClass' ] AS deviceClass," +
" parsedUserAgent['AgentNameVersionMajor'] AS agentNameVersionMajor," +
" expectedDeviceClass," +
" expectedAgentNameVersionMajor" +
" FROM ( "+
" SELECT " +
" eventTime, " +
" ParseUserAgent(useragent) AS parsedUserAgent," +
" expectedDeviceClass," +
" expectedAgentNameVersionMajor" +
" FROM AgentStream " +
" )" +
")" +
"GROUP BY TUMBLE(eventTime, INTERVAL '%d' %s), " +
" deviceClass," +
" agentNameVersionMajor," +
" expectedDeviceClass," +
" expectedAgentNameVersionMajor",
windowIntervalCount, windowIntervalScale,
windowIntervalCount, windowIntervalScale
);
Table resultTable = tableEnv.sqlQuery(sqlQuery);
TypeInformation<Row> tupleType = new RowTypeInfo(SQL_TIMESTAMP, STRING, STRING, STRING, STRING, LONG);
DataStream<Row> resultSet = tableEnv.toAppendStream(resultTable, tupleType);
resultSet.print();
resultSet.map((MapFunction<Row, String>) row -> {
Object useragent = row.getField(0);
Object deviceClass = row.getField(1);
Object agentNameVersionMajor = row.getField(2);
Object expectedDeviceClass = row.getField(3);
Object expectedAgentNameVersionMajor = row.getField(4);
assertEquals(
expectedDeviceClass,
deviceClass,
"Wrong DeviceClass: " + useragent);
assertEquals(
expectedAgentNameVersionMajor,
agentNameVersionMajor,
"Wrong AgentNameVersionMajor: " + useragent);
return useragent.toString();
});
senv.execute();
}
Example 14
| Source File: PopularPlacesSql.java From flink-training-exercises with Apache License 2.0 | 4 votes |
|
public static void main(String[] args) throws Exception {
// read parameters
ParameterTool params = ParameterTool.fromArgs(args);
String input = params.getRequired("input");
final int maxEventDelay = 60; // events are out of order by max 60 seconds
final int servingSpeedFactor = 600; // events of 10 minutes are served in 1 second
// set up streaming execution environment
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
// create a TableEnvironment
StreamTableEnvironment tEnv = StreamTableEnvironment.create(env);
// register TaxiRideTableSource as table "TaxiRides"
tEnv.registerTableSource(
"TaxiRides",
new TaxiRideTableSource(
input,
maxEventDelay,
servingSpeedFactor));
// register user-defined functions
tEnv.registerFunction("isInNYC", new GeoUtils.IsInNYC());
tEnv.registerFunction("toCellId", new GeoUtils.ToCellId());
tEnv.registerFunction("toCoords", new GeoUtils.ToCoords());
Table results = tEnv.sqlQuery(
"SELECT " +
"toCoords(cell), wstart, wend, isStart, popCnt " +
"FROM " +
"(SELECT " +
"cell, " +
"isStart, " +
"HOP_START(eventTime, INTERVAL '5' MINUTE, INTERVAL '15' MINUTE) AS wstart, " +
"HOP_END(eventTime, INTERVAL '5' MINUTE, INTERVAL '15' MINUTE) AS wend, " +
"COUNT(isStart) AS popCnt " +
"FROM " +
"(SELECT " +
"eventTime, " +
"isStart, " +
"CASE WHEN isStart THEN toCellId(startLon, startLat) ELSE toCellId(endLon, endLat) END AS cell " +
"FROM TaxiRides " +
"WHERE isInNYC(startLon, startLat) AND isInNYC(endLon, endLat)) " +
"GROUP BY cell, isStart, HOP(eventTime, INTERVAL '5' MINUTE, INTERVAL '15' MINUTE)) " +
"WHERE popCnt > 20"
);
// convert Table into an append stream and print it
// (if instead we needed a retraction stream we would use tEnv.toRetractStream)
tEnv.toAppendStream(results, Row.class).print();
// execute query
env.execute();
}
Example 15
| Source File: PopularPlacesTableApi.java From flink-training-exercises with Apache License 2.0 | 4 votes |
|
public static void main(String[] args) throws Exception {
// read parameters
ParameterTool params = ParameterTool.fromArgs(args);
String input = params.getRequired("input");
final int maxEventDelay = 60; // events are out of order by max 60 seconds
final int servingSpeedFactor = 600; // events of 10 minutes are served in 1 second
// set up streaming execution environment
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
// create a TableEnvironment
StreamTableEnvironment tEnv = StreamTableEnvironment.create(env);
// register TaxiRideTableSource as table "TaxiRides"
tEnv.registerTableSource(
"TaxiRides",
new TaxiRideTableSource(
input,
maxEventDelay,
servingSpeedFactor));
// register user-defined functions
tEnv.registerFunction("isInNYC", new GeoUtils.IsInNYC());
tEnv.registerFunction("toCellId", new GeoUtils.ToCellId());
tEnv.registerFunction("toCoords", new GeoUtils.ToCoords());
Table popPlaces = tEnv
// scan TaxiRides table
.scan("TaxiRides")
// filter for valid rides
.filter("isInNYC(startLon, startLat) && isInNYC(endLon, endLat)")
// select fields and compute grid cell of departure or arrival coordinates
.select("eventTime, " +
"isStart, " +
"(isStart = true).?(toCellId(startLon, startLat), toCellId(endLon, endLat)) AS cell")
// specify sliding window over 15 minutes with slide of 5 minutes
.window(Slide.over("15.minutes").every("5.minutes").on("eventTime").as("w"))
// group by cell, isStart, and window
.groupBy("cell, isStart, w")
// count departures and arrivals per cell (location) and window (time)
.select("cell, isStart, w.start AS start, w.end AS end, count(isStart) AS popCnt")
// filter for popular places
.filter("popCnt > 20")
// convert cell back to coordinates
.select("toCoords(cell) AS location, start, end, isStart, popCnt");
// convert Table into an append stream and print it
tEnv.toAppendStream(popPlaces, Row.class).print();
// execute query
env.execute();
}
Example 16
| Source File: RidesPerHour.java From flink-training-exercises with Apache License 2.0 | 4 votes |
|
public static void main(String[] args) throws Exception {
// read parameters
ParameterTool params = ParameterTool.fromArgs(args);
String input = params.getRequired("input");
final int maxEventDelay = 60; // events are out of order by max 60 seconds
final int servingSpeedFactor = 1800; // events of 30 minutes are served in 1 second
// set up streaming execution environment
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
// create a TableEnvironment
StreamTableEnvironment tEnv = StreamTableEnvironment.create(env);
// register TaxiRideTableSource as table "TaxiRides"
tEnv.registerTableSource(
"TaxiRides",
new TaxiRideTableSource(
input,
maxEventDelay,
servingSpeedFactor));
// register user-defined functions
tEnv.registerFunction("isInNYC", new GeoUtils.IsInNYC());
tEnv.registerFunction("toCellId", new GeoUtils.ToCellId());
tEnv.registerFunction("toCoords", new GeoUtils.ToCoords());
Table results = tEnv.sqlQuery(
//"SELECT TUMBLE_START(eventTime, INTERVAL '1' HOUR), isStart, count(isStart) FROM TaxiRides GROUP BY isStart, TUMBLE(eventTime, INTERVAL '1' HOUR)"
//"SELECT avg(endTime - startTime), passengerCnt FROM TaxiRides GROUP BY passengerCnt"
"SELECT CAST (toCellId(endLon, endLat) AS VARCHAR), eventTime," +
"COUNT(*) OVER (" +
"PARTITION BY toCellId(endLon, endLat) ORDER BY eventTime RANGE BETWEEN INTERVAL '10' MINUTE PRECEDING AND CURRENT ROW" +
") " +
"FROM( SELECT * FROM TaxiRides WHERE not isStart AND toCellId(endLon, endLat) = 50801 )"
);
// convert Table into an append stream and print it
// (if instead we needed a retraction stream we would use tEnv.toRetractStream)
tEnv.toRetractStream(results, Row.class).print();
// execute query
env.execute();
}
