org.apache.flink.table.api.Table Java Examples

public static void main(String[] args) throws Exception {
    ExecutionEnvironment environment = ExecutionEnvironment.getExecutionEnvironment();
    BatchTableEnvironment tableEnvironment = BatchTableEnvironment.getTableEnvironment(environment);
    String filepath = "file:\\D:\\imooc\\新一代大数据计算引擎 Flink从入门到实战-v\\input\\sales.csv";
    //csv => DataSet
    DataSource<Sales> csv = environment.readCsvFile(filepath)
            .ignoreFirstLine()
            .pojoType(Sales.class, "transactionId", "customerId", "itemId", "amountPaid");
    //csv.print();

    Table sales = tableEnvironment.fromDataSet(csv);
    tableEnvironment.registerTable("sales", sales);
    Table resultTable = tableEnvironment.sqlQuery("select customerId, sum(amountPaid) money from sales group by customerId");

    DataSet<Row> result = tableEnvironment.toDataSet(resultTable, Row.class);
    result.print();
}
 

@Test
public void test() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	StreamTableEnvironment tableEnv = createTableEnv(env);

	DataStream<Tuple2<Long, String>> ds = env.fromElements(
			Tuple2.of(1L, "a"),
			Tuple2.of(2L, "b"),
			Tuple2.of(3L, "c")
	);

	Table table = tableEnv.fromDataStream(ds, "id, name");

	Table result = tableEnv.sqlQuery("SELECT * from " + table);

	tableEnv.toAppendStream(result, ds.getType()).print();
	env.execute("test");
}
 

@Test
public void testAsFromAndToTuple() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env, config());

	Table table = tableEnv
		.fromDataSet(CollectionDataSets.get3TupleDataSet(env), "a, b, c")
		.select("a, b, c");

	TypeInformation<?> ti = new TupleTypeInfo<Tuple3<Integer, Long, String>>(
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.LONG_TYPE_INFO,
		BasicTypeInfo.STRING_TYPE_INFO);

	DataSet<?> ds = tableEnv.toDataSet(table, ti);
	List<?> results = ds.collect();
	String expected = "(1,1,Hi)\n" + "(2,2,Hello)\n" + "(3,2,Hello world)\n" +
		"(4,3,Hello world, how are you?)\n" + "(5,3,I am fine.)\n" + "(6,3,Luke Skywalker)\n" +
		"(7,4,Comment#1)\n" + "(8,4,Comment#2)\n" + "(9,4,Comment#3)\n" + "(10,4,Comment#4)\n" +
		"(11,5,Comment#5)\n" + "(12,5,Comment#6)\n" + "(13,5,Comment#7)\n" +
		"(14,5,Comment#8)\n" + "(15,5,Comment#9)\n" + "(16,6,Comment#10)\n" +
		"(17,6,Comment#11)\n" + "(18,6,Comment#12)\n" + "(19,6,Comment#13)\n" +
		"(20,6,Comment#14)\n" + "(21,6,Comment#15)\n";
	compareResultAsText(results, expected);
}
 

@Test
public void testAsFromAndToPrivateFieldPojo() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env, config());

	List<PrivateSmallPojo> data = new ArrayList<>();
	data.add(new PrivateSmallPojo("Peter", 28, 4000.00, "Sales"));
	data.add(new PrivateSmallPojo("Anna", 56, 10000.00, "Engineering"));
	data.add(new PrivateSmallPojo("Lucy", 42, 6000.00, "HR"));

	Table table = tableEnv
		.fromDataSet(env.fromCollection(data),
			"department AS a, " +
			"age AS b, " +
			"salary AS c, " +
			"name AS d")
		.select("a, b, c, d");

	DataSet<PrivateSmallPojo2> ds = tableEnv.toDataSet(table, PrivateSmallPojo2.class);
	List<PrivateSmallPojo2> results = ds.collect();
	String expected =
		"Sales,28,4000.0,Peter\n" +
		"Engineering,56,10000.0,Anna\n" +
		"HR,42,6000.0,Lucy\n";
	compareResultAsText(results, expected);
}
 

@Test
public void testAsFromAndToTuple() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env, config());

	Table table = tableEnv
		.fromDataSet(CollectionDataSets.get3TupleDataSet(env), $("a"), $("b"), $("c"))
		.select($("a"), $("b"), $("c"));

	TypeInformation<?> ti = new TupleTypeInfo<Tuple3<Integer, Long, String>>(
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.LONG_TYPE_INFO,
		BasicTypeInfo.STRING_TYPE_INFO);

	DataSet<?> ds = tableEnv.toDataSet(table, ti);
	List<?> results = ds.collect();
	String expected = "(1,1,Hi)\n" + "(2,2,Hello)\n" + "(3,2,Hello world)\n" +
		"(4,3,Hello world, how are you?)\n" + "(5,3,I am fine.)\n" + "(6,3,Luke Skywalker)\n" +
		"(7,4,Comment#1)\n" + "(8,4,Comment#2)\n" + "(9,4,Comment#3)\n" + "(10,4,Comment#4)\n" +
		"(11,5,Comment#5)\n" + "(12,5,Comment#6)\n" + "(13,5,Comment#7)\n" +
		"(14,5,Comment#8)\n" + "(15,5,Comment#9)\n" + "(16,6,Comment#10)\n" +
		"(17,6,Comment#11)\n" + "(18,6,Comment#12)\n" + "(19,6,Comment#13)\n" +
		"(20,6,Comment#14)\n" + "(21,6,Comment#15)\n";
	compareResultAsText(results, expected);
}
 

@Test
public void testFilter() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env);
	StreamITCase.clear();

	DataStream<Tuple5<Integer, Long, Integer, String, Long>> ds = JavaStreamTestData.get5TupleDataStream(env);
	tableEnv.registerDataStream("MyTable", ds, "a, b, c, d, e");

	String sqlQuery = "SELECT a, b, e FROM MyTable WHERE c < 4";
	Table result = tableEnv.sqlQuery(sqlQuery);

	DataStream<Row> resultSet = tableEnv.toAppendStream(result, Row.class);
	resultSet.addSink(new StreamITCase.StringSink<Row>());
	env.execute();

	List<String> expected = new ArrayList<>();
	expected.add("1,1,1");
	expected.add("2,2,2");
	expected.add("2,3,1");
	expected.add("3,4,2");

	StreamITCase.compareWithList(expected);
}
 

@Test
public void testAsFromTupleByPosition() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env, config());

	Table table = tableEnv
		.fromDataSet(CollectionDataSets.get3TupleDataSet(env), $("a"), $("b"), $("c"))
		.select($("a"), $("b"), $("c"));

	DataSet<Row> ds = tableEnv.toDataSet(table, Row.class);
	List<Row> results = ds.collect();
	String expected = "1,1,Hi\n" + "2,2,Hello\n" + "3,2,Hello world\n" +
		"4,3,Hello world, how are you?\n" + "5,3,I am fine.\n" + "6,3,Luke Skywalker\n" +
		"7,4,Comment#1\n" + "8,4,Comment#2\n" + "9,4,Comment#3\n" + "10,4,Comment#4\n" +
		"11,5,Comment#5\n" + "12,5,Comment#6\n" + "13,5,Comment#7\n" +
		"14,5,Comment#8\n" + "15,5,Comment#9\n" + "16,6,Comment#10\n" +
		"17,6,Comment#11\n" + "18,6,Comment#12\n" + "19,6,Comment#13\n" +
		"20,6,Comment#14\n" + "21,6,Comment#15\n";
	compareResultAsText(results, expected);
}
 

@Test
public void testAsFromPojoProjected() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env, config());

	List<SmallPojo> data = new ArrayList<>();
	data.add(new SmallPojo("Peter", 28, 4000.00, "Sales", new Integer[] {42}));
	data.add(new SmallPojo("Anna", 56, 10000.00, "Engineering", new Integer[] {}));
	data.add(new SmallPojo("Lucy", 42, 6000.00, "HR", new Integer[] {1, 2, 3}));

	Table table = tableEnv
		.fromDataSet(env.fromCollection(data), "name AS d")
		.select("d");

	DataSet<Row> ds = tableEnv.toDataSet(table, Row.class);
	List<Row> results = ds.collect();
	String expected =
		"Peter\n" +
		"Anna\n" +
		"Lucy\n";
	compareResultAsText(results, expected);
}
 

public static void main(String[] args) throws Exception {
    StreamExecutionEnvironment blinkStreamEnv = StreamExecutionEnvironment.getExecutionEnvironment();
    blinkStreamEnv.setParallelism(1);
    EnvironmentSettings blinkStreamSettings = EnvironmentSettings.newInstance()
            .useBlinkPlanner()
            .inStreamingMode()
            .build();
    StreamTableEnvironment blinkStreamTableEnv = StreamTableEnvironment.create(blinkStreamEnv, blinkStreamSettings);

    ParameterTool parameterTool = ExecutionEnvUtil.PARAMETER_TOOL;
    Properties properties = KafkaConfigUtil.buildKafkaProps(parameterTool);
    DataStream<String> dataStream = blinkStreamEnv.addSource(new FlinkKafkaConsumer011<>(parameterTool.get("kafka.topic"), new SimpleStringSchema(), properties));
    Table table = blinkStreamTableEnv.fromDataStream(dataStream, "word");
    blinkStreamTableEnv.registerTable("kafkaDataStream", table);

    RetractStreamTableSink<Row> retractStreamTableSink = new MyRetractStreamTableSink(new String[]{"_count", "word"}, new DataType[]{DataTypes.BIGINT(), DataTypes.STRING()});
    blinkStreamTableEnv.registerTableSink("sinkTable", retractStreamTableSink);

    Table wordCount = blinkStreamTableEnv.sqlQuery("SELECT count(word) AS _count,word FROM kafkaDataStream GROUP BY word");

    wordCount.insertInto("sinkTable");

    blinkStreamTableEnv.execute("Blink Kafka Table Source");
}
 

@Test
public void testRetractStreamDoesNotOverwriteTableConfig() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	DataStreamSource<Integer> elements = env.fromElements(1, 2, 3);

	StreamTableEnvironmentImpl tEnv = getStreamTableEnvironment(env, elements);

	Time minRetention = Time.minutes(1);
	Time maxRetention = Time.minutes(10);
	tEnv.getConfig().setIdleStateRetentionTime(minRetention, maxRetention);
	Table table = tEnv.fromDataStream(elements);
	tEnv.toRetractStream(table, Row.class);

	assertThat(
		tEnv.getConfig().getMinIdleStateRetentionTime(),
		equalTo(minRetention.toMilliseconds()));
	assertThat(
		tEnv.getConfig().getMaxIdleStateRetentionTime(),
		equalTo(maxRetention.toMilliseconds()));
}
 

@Test
public void testForceTypeWithTableSchema() {
	ExecutionEnvironment env = MLEnvironmentFactory.getDefault().getExecutionEnvironment();

	DataSet<Row> input = env.fromElements(Row.of("s1")).map(new GenericTypeMap());
	Table table2 = DataSetConversionUtil.toTable(MLEnvironmentFactory.DEFAULT_ML_ENVIRONMENT_ID,
		input,
		new TableSchema(
			new String[]{"word"},
			new TypeInformation[]{TypeInformation.of(Integer.class)}
		)
	);
	Assert.assertEquals(
		new TableSchema(new String[] {"word"}, new TypeInformation[] {TypeInformation.of(Integer.class)}),
		table2.getSchema()
	);

}
 

/**
 * Save the word-topic model in the sideOutputs.
 */
private void saveWordTopicModelAndPerplexity(DataSet<Row> model, int numTopic,
                                             Boolean ifOnline) {
    DataSet<Row> wordTopicDataSet;
    if (ifOnline) {
        wordTopicDataSet = model.mapPartition(new BuildWordTopicModelOnline()).setParallelism(1);
    } else {
        wordTopicDataSet = model.mapPartition(new BuildWordTopicModelGibbs()).setParallelism(1);
    }
    String[] colNames = new String[numTopic + 1];
    TypeInformation[] colTypes = new TypeInformation[colNames.length];
    colNames[0] = "word";
    colTypes[0] = Types.STRING;
    for (int i = 0; i < numTopic; i++) {
        colNames[1 + i] = "topic_" + i;
        colTypes[1 + i] = Types.DOUBLE;
    }

    DataSet<Row> logPerplexity = model.mapPartition(new CalculatePerplexityAndLikelihood()).setParallelism(1);
    this.setSideOutputTables(new Table[] {
        DataSetConversionUtil.toTable(getMLEnvironmentId(), wordTopicDataSet, colNames, colTypes),
        DataSetConversionUtil.toTable(getMLEnvironmentId(),
            logPerplexity, new String[]{"logPerplexity", "logLikelihood"},
            new TypeInformation[]{Types.DOUBLE, Types.DOUBLE})
    });
}
 

/**
 * build chi-square test result, it is for table and vector.
 */
public static Table buildResult(DataSet<Row> in,
                                String[] selectedColNames,
                                Long sessionId) {

    String[] outColNames = new String[]{"col", "chisquare_test"};
    TypeInformation[] outColTypes;
    if (selectedColNames == null) {
        outColTypes = new TypeInformation[]{Types.LONG, Types.STRING};
    } else {
        outColTypes = new TypeInformation[]{Types.STRING, Types.STRING};
    }
    return DataSetConversionUtil.toTable(sessionId,
        in.map(new BuildResult(selectedColNames)),
        outColNames,
        outColTypes);
}
 

@Test
public void testFullScan() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment batchTableEnvironment = BatchTableEnvironment.create(env);
	ParquetTableSource tableSource = createParquetTableSource(testPath);
	((TableEnvironmentInternal) batchTableEnvironment).registerTableSourceInternal("ParquetTable", tableSource);
	String query =
		"SELECT foo " +
		"FROM ParquetTable";

	Table table = batchTableEnvironment.sqlQuery(query);
	DataSet<Row> dataSet = batchTableEnvironment.toDataSet(table, Row.class);
	List<Row> result = dataSet.collect();

	assertEquals(1000, result.size());
}
 

@Test
public void testSelect() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env);
	StreamITCase.clear();

	DataStream<Tuple3<Integer, Long, String>> ds = JavaStreamTestData.getSmall3TupleDataSet(env);
	Table in = tableEnv.fromDataStream(ds, "a,b,c");
	tableEnv.registerTable("MyTable", in);

	String sqlQuery = "SELECT * FROM MyTable";
	Table result = tableEnv.sqlQuery(sqlQuery);

	DataStream<Row> resultSet = tableEnv.toAppendStream(result, Row.class);
	resultSet.addSink(new StreamITCase.StringSink<Row>());
	env.execute();

	List<String> expected = new ArrayList<>();
	expected.add("1,1,Hi");
	expected.add("2,2,Hello");
	expected.add("3,2,Hello world");

	StreamITCase.compareWithList(expected);
}
 

@Test
public void testBatchTableSourceTableAPI() throws Exception {

	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env, config());
	BatchTableSource csvTable = CommonTestData.getCsvTableSource();

	((TableEnvironmentInternal) tableEnv).registerTableSourceInternal("persons", csvTable);

	Table result = tableEnv.scan("persons")
		.select($("id"), $("first"), $("last"), $("score"));

	DataSet<Row> resultSet = tableEnv.toDataSet(result, Row.class);
	List<Row> results = resultSet.collect();

	String expected = "1,Mike,Smith,12.3\n" +
		"2,Bob,Taylor,45.6\n" +
		"3,Sam,Miller,7.89\n" +
		"4,Peter,Smith,0.12\n" +
		"5,Liz,Williams,34.5\n" +
		"6,Sally,Miller,6.78\n" +
		"7,Alice,Smith,90.1\n" +
		"8,Kelly,Williams,2.34\n";

	compareResultAsText(results, expected);
}
 

@Test
public void testTableRegister() throws Exception {
	final String tableName = "MyTable";
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env, config());

	DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
	Table t = tableEnv.fromDataSet(ds);
	tableEnv.createTemporaryView(tableName, t);
	Table result = tableEnv.scan(tableName).select($("f0"), $("f1")).filter($("f0").isGreater(7));

	DataSet<Row> resultSet = tableEnv.toDataSet(result, Row.class);
	List<Row> results = resultSet.collect();
	String expected = "8,4\n" + "9,4\n" + "10,4\n" + "11,5\n" + "12,5\n" +
			"13,5\n" + "14,5\n" + "15,5\n" +
			"16,6\n" + "17,6\n" + "18,6\n" + "19,6\n" + "20,6\n" + "21,6\n";
	compareResultAsText(results, expected);
}
 

@Test
public void testInsertIntoDynamicPartition() throws Exception {
	String dbName = "default";
	String tblName = "dest";
	RowTypeInfo rowTypeInfo = createDestTable(dbName, tblName, 1);
	ObjectPath tablePath = new ObjectPath(dbName, tblName);

	TableEnvironment tableEnv = HiveTestUtils.createTableEnv();

	List<Row> toWrite = generateRecords(5);
	Table src = tableEnv.fromTableSource(new CollectionTableSource(toWrite, rowTypeInfo));
	tableEnv.registerTable("src", src);

	tableEnv.registerCatalog("hive", hiveCatalog);
	tableEnv.sqlQuery("select * from src").insertInto("hive", "default", "dest");
	tableEnv.execute("mytest");

	List<CatalogPartitionSpec> partitionSpecs = hiveCatalog.listPartitions(tablePath);
	assertEquals(toWrite.size(), partitionSpecs.size());

	verifyWrittenData(toWrite, hiveShell.executeQuery("select * from " + tblName));

	hiveCatalog.dropTable(tablePath, false);
}
 

public static void main(String[] args) throws Exception {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    BatchTableEnvironment tEnv = BatchTableEnvironment.create(env);

    DataSet<WC> input = env.fromElements(
            new WC("Hello", 1),
            new WC("zhisheng", 1),
            new WC("Hello", 1));

    tEnv.registerDataSet("WordCount", input, "word, c");

    Table table = tEnv.sqlQuery(
            "SELECT word, SUM(c) as c FROM WordCount GROUP BY word");   //注意，之前 WC 定义的是 count，但在 1.9 中 count 是关键字，所以会抛异常，改成 c ok

    DataSet<WC> result = tEnv.toDataSet(table, WC.class);

    result.print();
}
 

@Test
public void testValuesFromMixedObjectsAndExpressions() {
	JavaStreamTableTestUtil util = javaStreamTestUtil();
	Table t = util.getTableEnv().fromValues(
		row(1, "ABC", null),
		Row.of(Math.PI, "ABC", 1),
		Row.of(3.1f, "DEF", 2),
		row(99L, "DEFG", nullOf(DataTypes.INT())),
		Row.of(0d, "D", 4)
	);
	util.verifyPlan(t);
}
 

private void registerSource(StreamTableEnvironment env, SqlSubmitFlinkRequest request,
                            Map<String, TableSource> tableSources, Map<String, SourceDescriptor> sideSources) {
    request.getSources().forEach(consumer -> {
        try {
            TableType tableType = consumer.getTableType();
            switch (tableType) {
                case SIDE:
                    sideSources.put(consumer.getName(), consumer);
                    LOGGER.info("register side table, name:{}, class:{}", consumer.getName(), consumer.getClass());
                    break;
                case VIEW:
                    String sql = consumer.getSql();
                    Table table = registerSql(env, sql, tableSources, sideSources);
                    env.registerTable(consumer.getName(), table);
                    LOGGER.info("register view, name:{}", consumer.getName());
                    break;
                case TABLE:
                    TableSource tableSource = consumer.transform();
                    env.registerTableSource(consumer.getName(), tableSource);
                    tableSources.put(consumer.getName(), tableSource);
                    LOGGER.info("register table, name:{}, class:{}", consumer.getName(), tableSource.getClass());
                    break;
                default:
                    throw new UnsupportedOperationException("Unknow tableType" + tableType);
            }
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    });
}
 

@Test
public void testTableSourceFieldOrder() throws Exception {
	TableEnvironment tEnv = createBatchTableEnv();
	HBaseTableSource hbaseTable = new HBaseTableSource(getConf(), TEST_TABLE_1);
	// shuffle order of column registration
	hbaseTable.addColumn(FAMILY2, F2COL1, String.class);
	hbaseTable.addColumn(FAMILY3, F3COL1, Double.class);
	hbaseTable.addColumn(FAMILY1, F1COL1, Integer.class);
	hbaseTable.addColumn(FAMILY2, F2COL2, Long.class);
	hbaseTable.addColumn(FAMILY3, F3COL2, Boolean.class);
	hbaseTable.addColumn(FAMILY3, F3COL3, String.class);
	tEnv.registerTableSource("hTable", hbaseTable);

	Table table = tEnv.sqlQuery("SELECT * FROM hTable AS h");

	List<Row> results = collectBatchResult(table);
	String expected =
		"Hello-1,100,1.01,false,Welt-1,10\n" +
			"Hello-2,200,2.02,true,Welt-2,20\n" +
			"Hello-3,300,3.03,false,Welt-3,30\n" +
			"null,400,4.04,true,Welt-4,40\n" +
			"Hello-5,500,5.05,false,Welt-5,50\n" +
			"Hello-6,600,6.06,true,Welt-6,60\n" +
			"Hello-7,700,7.07,false,Welt-7,70\n" +
			"null,800,8.08,true,Welt-8,80\n";

	TestBaseUtils.compareResultAsText(results, expected);
}
 

@Override
public StatementSet addInsert(String targetPath, Table table, boolean overwrite) {
	UnresolvedIdentifier unresolvedIdentifier = tableEnvironment.getParser().parseIdentifier(targetPath);
	ObjectIdentifier objectIdentifier = tableEnvironment.getCatalogManager()
			.qualifyIdentifier(unresolvedIdentifier);

	operations.add(new CatalogSinkModifyOperation(
			objectIdentifier,
			table.getQueryOperation(),
			Collections.emptyMap(),
			overwrite,
			Collections.emptyMap()));

	return this;
}
 

public static void main(String[] args) throws Exception {
    StreamExecutionEnvironment blinkStreamEnv = StreamExecutionEnvironment.getExecutionEnvironment();
    blinkStreamEnv.setParallelism(1);
    EnvironmentSettings blinkStreamSettings = EnvironmentSettings.newInstance()
            .useBlinkPlanner()
            .inStreamingMode()
            .build();
    StreamTableEnvironment blinkStreamTableEnv = StreamTableEnvironment.create(blinkStreamEnv, blinkStreamSettings);

    String ddlSource = "CREATE TABLE user_behavior (\n" +
            "    user_id BIGINT,\n" +
            "    item_id BIGINT,\n" +
            "    category_id BIGINT,\n" +
            "    behavior STRING,\n" +
            "    ts TIMESTAMP(3)\n" +
            ") WITH (\n" +
            "    'connector.type' = 'kafka',\n" +
            "    'connector.version' = '0.11',\n" +
            "    'connector.topic' = 'user_behavior',\n" +
            "    'connector.startup-mode' = 'latest-offset',\n" +
            "    'connector.properties.zookeeper.connect' = 'localhost:2181',\n" +
            "    'connector.properties.bootstrap.servers' = 'localhost:9092',\n" +
            "    'format.type' = 'json'\n" +
            ")";

    String countSql = "select user_id, count(user_id) from user_behavior group by user_id";

    blinkStreamTableEnv.sqlUpdate(ddlSource);
    Table countTable = blinkStreamTableEnv.sqlQuery(countSql);
    blinkStreamTableEnv.toRetractStream(countTable, Row.class).print();

    String distinctSql = "select distinct(user_id) from user_behavior";
    Table distinctTable = blinkStreamTableEnv.sqlQuery(distinctSql);
    blinkStreamTableEnv.toRetractStream(distinctTable, Row.class).print("==");

    blinkStreamTableEnv.execute("Blink Stream SQL count/distinct demo");
}
 

@Test
public void testStructTypesWithJavaArray() throws Exception {
    StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
    see.setParallelism(1);
    StreamTableEnvironment tEnv = StreamTableEnvironment.create(see);

    String table = newTopic();

    sendTypedMessages(table, SchemaType.AVRO, faList, Optional.empty(), SchemaData.FA.class);

    tEnv
            .connect(getPulsarDescriptor(table))
            .inAppendMode()
            .registerTableSource(table);

    Table t = tEnv.scan(table).select("l");
    tEnv.toAppendStream(t, t.getSchema().toRowType())
            .map(new FailingIdentityMapper<Row>(faList.size()))
            .addSink(new SingletonStreamSink.StringSink<>()).setParallelism(1);

    try {
        see.execute("test struct in avro");
    } catch (Exception e) {

    }
    SingletonStreamSink.compareWithList(
            faList.subList(0, faList.size() - 1).stream().map(Objects::toString).collect(Collectors.toList()));
}
 

@Test
public void testFunction() {
	final TableEnvironment env = TableEnvironment.create(EnvironmentSettings.newInstance().build());
	final DataTypeFactory dataTypeFactory = ((TableEnvironmentInternal) env)
		.getCatalogManager()
		.getDataTypeFactory();

	final Table inputTable;
	if (testSpec.fieldDataTypes == null) {
		inputTable = env.fromValues(Row.of(testSpec.fieldData));
	} else {
		final DataTypes.UnresolvedField[] fields = IntStream.range(0, testSpec.fieldDataTypes.length)
			.mapToObj(i -> DataTypes.FIELD("f" + i, testSpec.fieldDataTypes[i]))
			.toArray(DataTypes.UnresolvedField[]::new);
		inputTable = env.fromValues(DataTypes.ROW(fields), Row.of(testSpec.fieldData));
	}

	for (TestItem testItem : testSpec.testItems) {
		try {
			if (testItem instanceof TableApiResultTestItem) {
				testTableApiResult(dataTypeFactory, inputTable, ((TableApiResultTestItem) testItem));
			} else if (testItem instanceof TableApiErrorTestItem) {
				testTableApiError(inputTable, ((TableApiErrorTestItem) testItem));
			} else if (testItem instanceof SqlResultTestItem) {
				testSqlResult(dataTypeFactory, env, inputTable, ((SqlResultTestItem) testItem));
			} else if (testItem instanceof SqlErrorTestItem) {
				testSqlError(env, inputTable, ((SqlErrorTestItem) testItem));
			}
		} catch (Throwable t) {
			throw new AssertionError("Failing test item: " + testItem.toString(), t);
		}
	}
}
 

@Override
public Table initializeDataSource() {
	return new CsvSourceBatchOp()
		.setMLEnvironmentId(getMLEnvironmentId())
		.setFilePath(getFilePath())
		.setFieldDelimiter("\n")
		.setQuoteChar(null)
		.setIgnoreFirstLine(getIgnoreFirstLine())
		.setSchemaStr(getTextCol() + " string")
		.getOutputTable();
}
 

@Test
public void testAvroStringAccess() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tEnv = BatchTableEnvironment.create(env, config());

	Table t = tEnv.fromDataSet(testData(env));
	Table result = t.select($("name"));
	List<Utf8> results = tEnv.toDataSet(result, Types.GENERIC(Utf8.class)).collect();
	String expected = "Charlie\n" +
			"Terminator\n" +
			"Whatever";
	TestBaseUtils.compareResultAsText(results, expected);
}
 

@Test
public void testAppend() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.getConfig().enableObjectReuse();
	env.getConfig().setParallelism(1);
	StreamTableEnvironment tEnv = StreamTableEnvironment.create(env);

	Table t = tEnv.fromDataStream(get4TupleDataStream(env), $("id"), $("num"), $("text"), $("ts"));

	tEnv.registerTable("T", t);

	tEnv.executeSql(
		"CREATE TABLE upsertSink (" +
			"  id INT," +
			"  num BIGINT," +
			"  ts TIMESTAMP(3)" +
			") WITH (" +
			"  'connector'='jdbc'," +
			"  'url'='" + DB_URL + "'," +
			"  'table-name'='" + OUTPUT_TABLE2 + "'" +
			")");

	TableResult tableResult = tEnv.executeSql(
		"INSERT INTO upsertSink SELECT id, num, ts FROM T WHERE id IN (2, 10, 20)");
	// wait to finish
	tableResult.getJobClient().get().getJobExecutionResult(Thread.currentThread().getContextClassLoader()).get();
	check(new Row[] {
		Row.of(2, 2, Timestamp.valueOf("1970-01-01 00:00:00.002")),
		Row.of(10, 4, Timestamp.valueOf("1970-01-01 00:00:00.01")),
		Row.of(20, 6, Timestamp.valueOf("1970-01-01 00:00:00.02"))
	}, DB_URL, OUTPUT_TABLE2, new String[]{"id", "num", "ts"});
}
 

/**
 * Creates a table using the given query in the given table environment.
 */
private Table createTable(TableEnvironment tableEnv, String selectQuery) {
	// parse and validate query
	try {
		return tableEnv.sqlQuery(selectQuery);
	} catch (Throwable t) {
		// catch everything such that the query does not crash the executor
		throw new SqlExecutionException("Invalid SQL statement.", t);
	}
}