Java Code Examples for org.apache.flink.table.api.java.StreamTableEnvironment#sql()
The following examples show how to use
org.apache.flink.table.api.java.StreamTableEnvironment#sql() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: FlinkAvroSQLClient.java From df_data_service with Apache License 2.0 | 5 votes |
|
public static void tcFlinkAvroSQL(String KafkaServerHostPort, String SchemaRegistryHostPort,
String srcTopic, String targetTopic,
String consumerGroupId, String sinkKeys, String sqlState) {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
StreamTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
Properties properties = new Properties();
properties.setProperty(ConstantApp.PK_KAFKA_HOST_PORT.replace("_", "."), KafkaServerHostPort);
properties.setProperty(ConstantApp.PK_KAFKA_CONSUMER_GROURP, consumerGroupId);
properties.setProperty(ConstantApp.PK_KAFKA_SCHEMA_REGISTRY_HOST_PORT.replace("_", "."), SchemaRegistryHostPort);
properties.setProperty(ConstantApp.PK_FLINK_TABLE_SINK_KEYS, sinkKeys);
String[] srcTopicList = srcTopic.split(",");
for (int i = 0; i < srcTopicList.length; i++) {
properties.setProperty(ConstantApp.PK_SCHEMA_SUB_INPUT, srcTopicList[i]);
properties.setProperty(ConstantApp.PK_SCHEMA_ID_INPUT, SchemaRegistryClient.getLatestSchemaIDFromProperty(properties, ConstantApp.PK_SCHEMA_SUB_INPUT) + "");
properties.setProperty(ConstantApp.PK_SCHEMA_STR_INPUT, SchemaRegistryClient.getLatestSchemaFromProperty(properties, ConstantApp.PK_SCHEMA_SUB_INPUT).toString());
tableEnv.registerTableSource(srcTopicList[i], new Kafka010AvroTableSource(srcTopicList[i], properties));
}
try {
Table result = tableEnv.sql(sqlState);
SchemaRegistryClient.addSchemaFromTableResult(SchemaRegistryHostPort, targetTopic, result);
// For old producer, we need to create topic-value subject as well
SchemaRegistryClient.addSchemaFromTableResult(SchemaRegistryHostPort, targetTopic + "-value", result);
// delivered properties for sink
properties.setProperty(ConstantApp.PK_SCHEMA_SUB_OUTPUT, targetTopic);
properties.setProperty(ConstantApp.PK_SCHEMA_ID_OUTPUT, SchemaRegistryClient.getLatestSchemaIDFromProperty(properties, ConstantApp.PK_SCHEMA_SUB_OUTPUT) + "");
properties.setProperty(ConstantApp.PK_SCHEMA_STR_OUTPUT, SchemaRegistryClient.getLatestSchemaFromProperty(properties, ConstantApp.PK_SCHEMA_SUB_OUTPUT).toString());
Kafka09AvroTableSink avro_sink =
new Kafka09AvroTableSink(targetTopic, properties, new FlinkFixedPartitioner());
result.writeToSink(avro_sink);
env.execute("DF_FlinkSQL_Client_" + srcTopic + "-" + targetTopic);
} catch (Exception e) {
e.printStackTrace();
}
}
Example 2
| Source File: UnitTestSuiteFlink.java From df_data_service with Apache License 2.0 | 5 votes |
|
public static void testFlinkAvroSQL() {
System.out.println("TestCase_Test Avro SQL");
String resultFile = "/home/vagrant/test.txt";
String jarPath = DFInitService.class.getProtectionDomain().getCodeSource().getLocation().getPath();
StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", 6123, jarPath)
.setParallelism(1);
StreamTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
Properties properties = new Properties();
properties.setProperty("bootstrap.servers", "localhost:9092");
properties.setProperty("group.id", "consumer_test");
properties.setProperty("schema.subject", "test-value");
properties.setProperty("schema.registry", "localhost:8081");
properties.setProperty("static.avro.schema", "empty_schema");
try {
Kafka09AvroTableSource kafkaAvroTableSource = new Kafka09AvroTableSource("test", properties);
tableEnv.registerTableSource("Orders", kafkaAvroTableSource);
//Table result = tableEnv.sql("SELECT STREAM name, symbol, exchange FROM Orders");
Table result = tableEnv.sql("SELECT name, symbol, exchangecode FROM Orders");
Files.deleteIfExists(Paths.get(resultFile));
// create a TableSink
TableSink sink = new CsvTableSink(resultFile, "|");
// write the result Table to the TableSink
result.writeToSink(sink);
env.execute("Flink AVRO SQL KAFKA Test");
} catch (Exception e) {
e.printStackTrace();
}
}
Example 3
| Source File: UnitTestSuiteFlink.java From df_data_service with Apache License 2.0 | 5 votes |
|
public static void testFlinkAvroSQLJson() {
System.out.println("TestCase_Test Avro SQL to Json Sink");
final String STATIC_USER_SCHEMA = "{"
+ "\"type\":\"record\","
+ "\"name\":\"myrecord\","
+ "\"fields\":["
+ " { \"name\":\"symbol\", \"type\":\"string\" },"
+ " { \"name\":\"name\", \"type\":\"string\" },"
+ " { \"name\":\"exchangecode\", \"type\":\"string\" }"
+ "]}";
String jarPath = DFInitService.class.getProtectionDomain().getCodeSource().getLocation().getPath();
DFRemoteStreamEnvironment env = new DFRemoteStreamEnvironment("localhost", 6123, jarPath)
.setParallelism(1);
StreamTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
Properties properties = new Properties();
properties.setProperty("bootstrap.servers", "localhost:9092");
properties.setProperty("group.id", "consumer_test");
properties.setProperty("schema.subject", "test-value");
properties.setProperty("schema.registry", "localhost:8081");
properties.setProperty("useAvro", "avro");
properties.setProperty("static.avro.schema",
SchemaRegistryClient.getSchemaFromRegistry("http://localhost:8081", "test-value", "latest").toString());
try {
HashMap<String, String> hm = new HashMap<>();
Kafka09AvroTableSource kafkaAvroTableSource = new Kafka09AvroTableSource("test", properties);
tableEnv.registerTableSource("Orders", kafkaAvroTableSource);
Table result = tableEnv.sql("SELECT name, symbol, exchangecode FROM Orders");
//Kafka09JsonTableSink json_sink = new Kafka09JsonTableSink ("test_json", properties, new FlinkFixedPartitioner());
Kafka09AvroTableSink json_sink = new Kafka09AvroTableSink ("test_json", properties, new FlinkFixedPartitioner());
// write the result Table to the TableSink
result.writeToSink(json_sink);
env.executeWithDFObj("Flink AVRO SQL KAFKA Test", new DFJobPOPJ().setJobConfig(hm) );
} catch (Exception e) {
e.printStackTrace();
}
}
Example 4
| Source File: TCFlinkAvroSQL.java From df_data_service with Apache License 2.0 | 4 votes |
|
public static void tcFlinkAvroSQL(String SchemaRegistryHostPort, String srcTopic, String targetTopic, String sqlState) {
System.out.println("tcFlinkAvroSQL");
String resultFile = "testResult";
String jarPath = "C:/Users/dadu/Coding/df_data_service/target/df-data-service-1.1-SNAPSHOT-fat.jar";
//String jarPath = "/Users/will/Documents/Coding/GitHub/df_data_service/target/df-data-service-1.1-SNAPSHOT-fat.jar";
StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", 6123, jarPath)
.setParallelism(1);
StreamTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
Properties properties = new Properties();
properties.setProperty(ConstantApp.PK_KAFKA_HOST_PORT.replace("_", "."), "localhost:9092");
properties.setProperty(ConstantApp.PK_KAFKA_CONSUMER_GROURP, "consumer_test");
//properties.setProperty(ConstantApp.PK_SCHEMA_SUB_OUTPUT, "test");
properties.setProperty(ConstantApp.PK_KAFKA_SCHEMA_REGISTRY_HOST_PORT.replace("_", "."), SchemaRegistryHostPort);
properties.setProperty(ConstantApp.PK_FLINK_TABLE_SINK_KEYS, "symbol");
String[] srcTopicList = srcTopic.split(",");
for (int i = 0; i < srcTopicList.length; i++) {
properties.setProperty(ConstantApp.PK_SCHEMA_SUB_INPUT, srcTopicList[i]);
properties.setProperty(ConstantApp.PK_SCHEMA_ID_INPUT, SchemaRegistryClient.getLatestSchemaIDFromProperty(properties, ConstantApp.PK_SCHEMA_SUB_INPUT) + "");
properties.setProperty(ConstantApp.PK_SCHEMA_STR_INPUT, SchemaRegistryClient.getLatestSchemaFromProperty(properties, ConstantApp.PK_SCHEMA_SUB_INPUT).toString());
tableEnv.registerTableSource(srcTopicList[i], new Kafka010AvroTableSource(srcTopicList[i], properties));
}
try {
Table result = tableEnv.sql(sqlState);
result.printSchema();
System.out.println("generated avro schema is = " + SchemaRegistryClient.tableAPIToAvroSchema(result, targetTopic));
SchemaRegistryClient.addSchemaFromTableResult(SchemaRegistryHostPort, targetTopic, result);
// delivered properties
properties.setProperty(ConstantApp.PK_SCHEMA_SUB_OUTPUT, targetTopic);
properties.setProperty(ConstantApp.PK_SCHEMA_ID_OUTPUT, SchemaRegistryClient.getLatestSchemaIDFromProperty(properties, ConstantApp.PK_SCHEMA_SUB_OUTPUT) + "");
properties.setProperty(ConstantApp.PK_SCHEMA_STR_OUTPUT, SchemaRegistryClient.getLatestSchemaFromProperty(properties, ConstantApp.PK_SCHEMA_SUB_OUTPUT).toString());
System.out.println(Paths.get(resultFile).toAbsolutePath());
Kafka09AvroTableSink avro_sink =
new Kafka09AvroTableSink(targetTopic, properties, new FlinkFixedPartitioner());
result.writeToSink(avro_sink);
//result.writeToSink(new CsvTableSink(resultFile, "|", 1, FileSystem.WriteMode.OVERWRITE));
env.execute("tcFlinkAvroSQL");
} catch (Exception e) {
e.printStackTrace();
}
}
Example 5
| Source File: UnitTestSuiteFlink.java From df_data_service with Apache License 2.0 | 4 votes |
|
public static void testFlinkSQL() {
LOG.info("Only Unit Testing Function is enabled");
String resultFile = "/home/vagrant/test.txt";
try {
String jarPath = DFInitService.class.getProtectionDomain().getCodeSource().getLocation().getPath();
StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", 6123, jarPath)
.setParallelism(1);
String kafkaTopic = "finance";
String kafkaTopic_stage = "df_trans_stage_finance";
String kafkaTopic_out = "df_trans_out_finance";
StreamTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
Properties properties = new Properties();
properties.setProperty("bootstrap.servers", "localhost:9092");
properties.setProperty("group.id", "consumer3");
// Internal covert Json String to Json - Begin
DataStream<String> stream = env
.addSource(new FlinkKafkaConsumer09<>(kafkaTopic, new SimpleStringSchema(), properties));
stream.map(new MapFunction<String, String>() {
@Override
public String map(String jsonString) throws Exception {
return jsonString.replaceAll("\\\\", "").replace("\"{", "{").replace("}\"","}");
}
}).addSink(new FlinkKafkaProducer09<String>("localhost:9092", kafkaTopic_stage, new SimpleStringSchema()));
// Internal covert Json String to Json - End
String[] fieldNames = new String[] {"name"};
Class<?>[] fieldTypes = new Class<?>[] {String.class};
Kafka09AvroTableSource kafkaTableSource = new Kafka09AvroTableSource(
kafkaTopic_stage,
properties,
fieldNames,
fieldTypes);
//kafkaTableSource.setFailOnMissingField(true);
tableEnv.registerTableSource("Orders", kafkaTableSource);
//Table result = tableEnv.sql("SELECT STREAM name FROM Orders");
Table result = tableEnv.sql("SELECT name FROM Orders");
Files.deleteIfExists(Paths.get(resultFile));
// create a TableSink
TableSink sink = new CsvTableSink(resultFile, "|");
// write the result Table to the TableSink
result.writeToSink(sink);
env.execute("FlinkConsumer");
} catch (Exception e) {
e.printStackTrace();
}
}
Example 6
| Source File: UnitTestSuiteFlink.java From df_data_service with Apache License 2.0 | 4 votes |
|
public static void testFlinkAvroSQLWithStaticSchema() {
System.out.println("TestCase_Test Avro SQL with static Schema");
final String STATIC_USER_SCHEMA = "{"
+ "\"type\":\"record\","
+ "\"name\":\"myrecord\","
+ "\"fields\":["
+ " { \"name\":\"symbol\", \"type\":\"string\" },"
+ " { \"name\":\"name\", \"type\":\"string\" },"
+ " { \"name\":\"exchangecode\", \"type\":\"string\" }"
+ "]}";
String resultFile = "/home/vagrant/test.txt";
String jarPath = DFInitService.class.getProtectionDomain().getCodeSource().getLocation().getPath();
StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", 6123, jarPath)
.setParallelism(1);
StreamTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
Properties properties = new Properties();
properties.setProperty("bootstrap.servers", "localhost:9092");
properties.setProperty("group.id", "consumer_test");
properties.setProperty("schema.subject", "test-value");
properties.setProperty("schema.registry", "localhost:8081");
properties.setProperty("static.avro.schema", STATIC_USER_SCHEMA);
try {
Kafka09AvroTableSource kafkaAvroTableSource = new Kafka09AvroTableSource("test", properties);
tableEnv.registerTableSource("Orders", kafkaAvroTableSource);
//Table result = tableEnv.sql("SELECT STREAM name, symbol, exchange FROM Orders");
Table result = tableEnv.sql("SELECT name, symbol, exchangecode FROM Orders");
Files.deleteIfExists(Paths.get(resultFile));
// create a TableSink
TableSink sink = new CsvTableSink(resultFile, "|");
// write the result Table to the TableSink
result.writeToSink(sink);
env.execute("Flink AVRO SQL KAFKA Test");
} catch (Exception e) {
e.printStackTrace();
}
}
