Java Code Examples for org.apache.flink.table.api.java.BatchTableEnvironment#create()

public static void main(String[] args) throws Exception {
    ExecutionEnvironment env = ExecutionEnvironment.createCollectionsEnvironment();
    BatchTableEnvironment tEnv = BatchTableEnvironment.create(env);

    DataSet<WC> input = env.fromElements(
            new WC("Hello", 1),
            new WC("zhisheng", 2),
            new WC("Hello", 1));

    Table table = tEnv.fromDataSet(input);

    Table filtered = table
            .groupBy("word")
            .select("word, c.sum as c")
            .filter("c = 2");

    DataSet<WC> result = tEnv.toDataSet(filtered, WC.class);

    result.print();
}
 

@Test
public void testAsFromAndToTuple() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env, config());

	Table table = tableEnv
		.fromDataSet(CollectionDataSets.get3TupleDataSet(env), "a, b, c")
		.select("a, b, c");

	TypeInformation<?> ti = new TupleTypeInfo<Tuple3<Integer, Long, String>>(
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.LONG_TYPE_INFO,
		BasicTypeInfo.STRING_TYPE_INFO);

	DataSet<?> ds = tableEnv.toDataSet(table, ti);
	List<?> results = ds.collect();
	String expected = "(1,1,Hi)\n" + "(2,2,Hello)\n" + "(3,2,Hello world)\n" +
		"(4,3,Hello world, how are you?)\n" + "(5,3,I am fine.)\n" + "(6,3,Luke Skywalker)\n" +
		"(7,4,Comment#1)\n" + "(8,4,Comment#2)\n" + "(9,4,Comment#3)\n" + "(10,4,Comment#4)\n" +
		"(11,5,Comment#5)\n" + "(12,5,Comment#6)\n" + "(13,5,Comment#7)\n" +
		"(14,5,Comment#8)\n" + "(15,5,Comment#9)\n" + "(16,6,Comment#10)\n" +
		"(17,6,Comment#11)\n" + "(18,6,Comment#12)\n" + "(19,6,Comment#13)\n" +
		"(20,6,Comment#14)\n" + "(21,6,Comment#15)\n";
	compareResultAsText(results, expected);
}
 

@Test
public void testAsFromAndToPrivateFieldPojo() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env, config());

	List<PrivateSmallPojo> data = new ArrayList<>();
	data.add(new PrivateSmallPojo("Peter", 28, 4000.00, "Sales"));
	data.add(new PrivateSmallPojo("Anna", 56, 10000.00, "Engineering"));
	data.add(new PrivateSmallPojo("Lucy", 42, 6000.00, "HR"));

	Table table = tableEnv
		.fromDataSet(env.fromCollection(data),
			"department AS a, " +
			"age AS b, " +
			"salary AS c, " +
			"name AS d")
		.select("a, b, c, d");

	DataSet<PrivateSmallPojo2> ds = tableEnv.toDataSet(table, PrivateSmallPojo2.class);
	List<PrivateSmallPojo2> results = ds.collect();
	String expected =
		"Sales,28,4000.0,Peter\n" +
		"Engineering,56,10000.0,Anna\n" +
		"HR,42,6000.0,Lucy\n";
	compareResultAsText(results, expected);
}
 

@Test
public void testValues() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env, config());

	String sqlQuery = "VALUES (1, 'Test', TRUE, DATE '1944-02-24', 12.4444444444444445)," +
		"(2, 'Hello', TRUE, DATE '1944-02-24', 12.666666665)," +
		"(3, 'World', FALSE, DATE '1944-12-24', 12.54444445)";
	Table result = tableEnv.sqlQuery(sqlQuery);

	DataSet<Row> resultSet = tableEnv.toDataSet(result, Row.class);

	List<Row> results = resultSet.collect();
	String expected = "3,World,false,1944-12-24,12.5444444500000000\n" +
		"2,Hello,true,1944-02-24,12.6666666650000000\n" +
		// Calcite converts to decimals and strings with equal length
		"1,Test ,true,1944-02-24,12.4444444444444445\n";
	compareResultAsText(results, expected);
}
 

@Test
public void testSelectFromTable() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env, config());

	DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
	Table in = tableEnv.fromDataSet(ds, "a,b,c");
	tableEnv.registerTable("T", in);

	String sqlQuery = "SELECT a, c FROM T";
	Table result = tableEnv.sqlQuery(sqlQuery);

	DataSet<Row> resultSet = tableEnv.toDataSet(result, Row.class);
	List<Row> results = resultSet.collect();
	String expected = "1,Hi\n" + "2,Hello\n" + "3,Hello world\n" +
		"4,Hello world, how are you?\n" + "5,I am fine.\n" + "6,Luke Skywalker\n" +
		"7,Comment#1\n" + "8,Comment#2\n" + "9,Comment#3\n" + "10,Comment#4\n" +
		"11,Comment#5\n" + "12,Comment#6\n" + "13,Comment#7\n" +
		"14,Comment#8\n" + "15,Comment#9\n" + "16,Comment#10\n" +
		"17,Comment#11\n" + "18,Comment#12\n" + "19,Comment#13\n" +
		"20,Comment#14\n" + "21,Comment#15\n";
	compareResultAsText(results, expected);
}
 

public static void main(String[] args) throws Exception {

		// set up execution environment
		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		BatchTableEnvironment tEnv = BatchTableEnvironment.create(env);

		DataSet<WC> input = env.fromElements(
			new WC("Hello", 1),
			new WC("Ciao", 1),
			new WC("Hello", 1));

		// register the DataSet as table "WordCount"
		tEnv.registerDataSet("WordCount", input, "word, frequency");

		// run a SQL query on the Table and retrieve the result as a new Table
		Table table = tEnv.sqlQuery(
			"SELECT word, SUM(frequency) as frequency FROM WordCount GROUP BY word");

		DataSet<WC> result = tEnv.toDataSet(table, WC.class);

		result.print();
	}
 

@Test
public void testRegisterWithFields() throws Exception {
	final String tableName = "MyTable";
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env, config());

	DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
	tableEnv.registerDataSet(tableName, ds, "a, b, c");
	Table t = tableEnv.scan(tableName);

	Table result = t.select("a, b, c");

	DataSet<Row> resultSet = tableEnv.toDataSet(result, Row.class);
	List<Row> results = resultSet.collect();
	String expected = "1,1,Hi\n" + "2,2,Hello\n" + "3,2,Hello world\n" +
			"4,3,Hello world, how are you?\n" + "5,3,I am fine.\n" + "6,3,Luke Skywalker\n" +
			"7,4,Comment#1\n" + "8,4,Comment#2\n" + "9,4,Comment#3\n" + "10,4,Comment#4\n" +
			"11,5,Comment#5\n" + "12,5,Comment#6\n" + "13,5,Comment#7\n" +
			"14,5,Comment#8\n" + "15,5,Comment#9\n" + "16,6,Comment#10\n" +
			"17,6,Comment#11\n" + "18,6,Comment#12\n" + "19,6,Comment#13\n" +
			"20,6,Comment#14\n" + "21,6,Comment#15\n";
	compareResultAsText(results, expected);
}
 

@Ignore
@Test
public void testAsFromTupleToPojo() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env, config());

	List<Tuple4<String, Integer, Double, String>> data = new ArrayList<>();
	data.add(new Tuple4<>("Rofl", 1, 1.0, "Hi"));
	data.add(new Tuple4<>("lol", 2, 1.0, "Hi"));
	data.add(new Tuple4<>("Test me", 4, 3.33, "Hello world"));

	Table table = tableEnv
		.fromDataSet(env.fromCollection(data), "q, w, e, r")
		.select("q as a, w as b, e as c, r as d");

	DataSet<SmallPojo2> ds = tableEnv.toDataSet(table, SmallPojo2.class);
	List<SmallPojo2> results = ds.collect();
	String expected = "Rofl,1,1.0,Hi\n" + "lol,2,1.0,Hi\n" + "Test me,4,3.33,Hello world\n";
	compareResultAsText(results, expected);
}
 

public static void main(String[] args) throws Exception {
	ExecutionEnvironment env   = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tEnv = BatchTableEnvironment.create(env);

	tEnv.registerTableSource("transactions", new BoundedTransactionTableSource());
	tEnv.registerTableSink("spend_report", new SpendReportTableSink());
	tEnv.registerFunction("truncateDateToHour", new TruncateDateToHour());

	tEnv
		.scan("transactions")
		.insertInto("spend_report");

	env.execute("Spend Report");
}
 

private EnvironmentInstance() {
	// create environments
	if (mergedEnv.getExecution().isStreamingExecution()) {
		streamExecEnv = createStreamExecutionEnvironment();
		execEnv = null;
		tableEnv = StreamTableEnvironment.create(streamExecEnv);
	} else if (mergedEnv.getExecution().isBatchExecution()) {
		streamExecEnv = null;
		execEnv = createExecutionEnvironment();
		tableEnv = BatchTableEnvironment.create(execEnv);
	} else {
		throw new SqlExecutionException("Unsupported execution type specified.");
	}

	// create query config
	queryConfig = createQueryConfig();

	// register table sources
	tableSources.forEach(tableEnv::registerTableSource);

	// register table sinks
	tableSinks.forEach(tableEnv::registerTableSink);

	// register user-defined functions
	registerFunctions();

	// register views and temporal tables in specified order
	mergedEnv.getTables().forEach((name, entry) -> {
		// if registering a view fails at this point,
		// it means that it accesses tables that are not available anymore
		if (entry instanceof ViewEntry) {
			final ViewEntry viewEntry = (ViewEntry) entry;
			registerView(viewEntry);
		} else if (entry instanceof TemporalTableEntry) {
			final TemporalTableEntry temporalTableEntry = (TemporalTableEntry) entry;
			registerTemporalTable(temporalTableEntry);
		}
	});
}
 

@Test
public void testTableSourceReadAsByteArray() throws Exception {

	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(4);
	BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env, new TableConfig());
	// fetch row2 from the table till the end
	HBaseTableSource hbaseTable = new HBaseTableSource(getConf(), TEST_TABLE);
	hbaseTable.addColumn(FAMILY2, F2COL1, byte[].class);
	hbaseTable.addColumn(FAMILY2, F2COL2, byte[].class);

	tableEnv.registerTableSource("hTable", hbaseTable);
	tableEnv.registerFunction("toUTF8", new ToUTF8());
	tableEnv.registerFunction("toLong", new ToLong());

	Table result = tableEnv.sqlQuery(
		"SELECT " +
			"  toUTF8(h.family2.col1), " +
			"  toLong(h.family2.col2) " +
			"FROM hTable AS h"
	);
	DataSet<Row> resultSet = tableEnv.toDataSet(result, Row.class);
	List<Row> results = resultSet.collect();

	String expected =
		"Hello-1,100\n" +
		"Hello-2,200\n" +
		"Hello-3,300\n" +
		"null,400\n" +
		"Hello-5,500\n" +
		"Hello-6,600\n" +
		"Hello-7,700\n" +
		"null,800\n";

	TestBaseUtils.compareResultAsText(results, expected);
}
 

@Test
public void testTableSourceFullScan() throws Exception {

	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(4);
	BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env, new TableConfig());
	HBaseTableSource hbaseTable = new HBaseTableSource(getConf(), TEST_TABLE);
	hbaseTable.addColumn(FAMILY1, F1COL1, Integer.class);
	hbaseTable.addColumn(FAMILY2, F2COL1, String.class);
	hbaseTable.addColumn(FAMILY2, F2COL2, Long.class);
	hbaseTable.addColumn(FAMILY3, F3COL1, Double.class);
	hbaseTable.addColumn(FAMILY3, F3COL2, Boolean.class);
	hbaseTable.addColumn(FAMILY3, F3COL3, String.class);
	tableEnv.registerTableSource("hTable", hbaseTable);

	Table result = tableEnv.sqlQuery(
		"SELECT " +
			"  h.family1.col1, " +
			"  h.family2.col1, " +
			"  h.family2.col2, " +
			"  h.family3.col1, " +
			"  h.family3.col2, " +
			"  h.family3.col3 " +
			"FROM hTable AS h"
	);
	DataSet<Row> resultSet = tableEnv.toDataSet(result, Row.class);
	List<Row> results = resultSet.collect();

	String expected =
		"10,Hello-1,100,1.01,false,Welt-1\n" +
		"20,Hello-2,200,2.02,true,Welt-2\n" +
		"30,Hello-3,300,3.03,false,Welt-3\n" +
		"40,null,400,4.04,true,Welt-4\n" +
		"50,Hello-5,500,5.05,false,Welt-5\n" +
		"60,Hello-6,600,6.06,true,Welt-6\n" +
		"70,Hello-7,700,7.07,false,Welt-7\n" +
		"80,null,800,8.08,true,Welt-8\n";

	TestBaseUtils.compareResultAsText(results, expected);
}
 

@Test
public void testBatchTableSinkUsingDescriptor() throws Exception {

    // create a Pravega stream for test purposes
    Stream stream = Stream.of(setupUtils.getScope(), "testBatchTableSinkUsingDescriptor");
    this.setupUtils.createTestStream(stream.getStreamName(), 1);

    // create a Flink Table environment
    ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
    env.setParallelism(1);
    BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env);

    Table table = tableEnv.fromDataSet(env.fromCollection(SAMPLES));

    Pravega pravega = new Pravega();
    pravega.tableSinkWriterBuilder()
            .withRoutingKeyField("category")
            .forStream(stream)
            .withPravegaConfig(setupUtils.getPravegaConfig());

    ConnectTableDescriptor desc = tableEnv.connect(pravega)
            .withFormat(new Json().failOnMissingField(true))
            .withSchema(new Schema().field("category", DataTypes.STRING()).
                    field("value", DataTypes.INT()));
    desc.createTemporaryTable("test");

    final Map<String, String> propertiesMap = desc.toProperties();
    final TableSink<?> sink = TableFactoryService.find(BatchTableSinkFactory.class, propertiesMap)
            .createBatchTableSink(propertiesMap);

    String tableSinkPath = tableEnv.getCurrentDatabase() + "." + "PravegaSink";

    ConnectorCatalogTable<?, ?> connectorCatalogSinkTable = ConnectorCatalogTable.sink(sink, true);

    tableEnv.getCatalog(tableEnv.getCurrentCatalog()).get().createTable(
            ObjectPath.fromString(tableSinkPath),
            connectorCatalogSinkTable, false);
    table.insertInto("PravegaSink");
    env.execute();
}
 

@Test(expected = TableException.class)
public void testGenericRowWithAlias() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env, config());

	// use null value the enforce GenericType
	DataSet<Row> dataSet = env.fromElements(Row.of((Integer) null));
	assertTrue(dataSet.getType() instanceof GenericTypeInfo);
	assertTrue(dataSet.getType().getTypeClass().equals(Row.class));

	// Must fail. Cannot import DataSet<Row> with GenericTypeInfo.
	tableEnv.fromDataSet(dataSet, "nullField");
}
 

@Test
public void testAsFromPojo() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env, config());

	List<SmallPojo> data = new ArrayList<>();
	data.add(new SmallPojo("Peter", 28, 4000.00, "Sales", new Integer[] {42}));
	data.add(new SmallPojo("Anna", 56, 10000.00, "Engineering", new Integer[] {}));
	data.add(new SmallPojo("Lucy", 42, 6000.00, "HR", new Integer[] {1, 2, 3}));

	Table table = tableEnv
		.fromDataSet(env.fromCollection(data),
			"department AS a, " +
			"age AS b, " +
			"salary AS c, " +
			"name AS d," +
			"roles as e")
		.select("a, b, c, d, e");

	DataSet<Row> ds = tableEnv.toDataSet(table, Row.class);
	List<Row> results = ds.collect();
	String expected =
		"Sales,28,4000.0,Peter,[42]\n" +
		"Engineering,56,10000.0,Anna,[]\n" +
		"HR,42,6000.0,Lucy,[1, 2, 3]\n";
	compareResultAsText(results, expected);
}
 

@Test(expected = TableException.class)
public void testRegisterTableFromOtherEnv() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tableEnv1 = BatchTableEnvironment.create(env, config());
	BatchTableEnvironment tableEnv2 = BatchTableEnvironment.create(env, config());

	Table t = tableEnv1.fromDataSet(CollectionDataSets.get3TupleDataSet(env));
	// Must fail. Table is bound to different TableEnvironment.
	tableEnv2.registerTable("MyTable", t);
}
 

@Test(expected = TableException.class)
public void testRegisterExistingDatasetTable() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env, config());

	DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
	tableEnv.registerDataSet("MyTable", ds);
	DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 =
			CollectionDataSets.getSmall5TupleDataSet(env);
	// Must fail. Name is already used for different table.
	tableEnv.registerDataSet("MyTable", ds2);
}
 

@Test
public void testTableSourceProjection() throws Exception {

	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(4);
	BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env, new TableConfig());
	HBaseTableSource hbaseTable = new HBaseTableSource(getConf(), TEST_TABLE);
	hbaseTable.addColumn(FAMILY1, F1COL1, Integer.class);
	hbaseTable.addColumn(FAMILY2, F2COL1, String.class);
	hbaseTable.addColumn(FAMILY2, F2COL2, Long.class);
	hbaseTable.addColumn(FAMILY3, F3COL1, Double.class);
	hbaseTable.addColumn(FAMILY3, F3COL2, Boolean.class);
	hbaseTable.addColumn(FAMILY3, F3COL3, String.class);
	tableEnv.registerTableSource("hTable", hbaseTable);

	Table result = tableEnv.sqlQuery(
		"SELECT " +
			"  h.family1.col1, " +
			"  h.family3.col1, " +
			"  h.family3.col2, " +
			"  h.family3.col3 " +
			"FROM hTable AS h"
	);
	DataSet<Row> resultSet = tableEnv.toDataSet(result, Row.class);
	List<Row> results = resultSet.collect();

	String expected =
		"10,1.01,false,Welt-1\n" +
		"20,2.02,true,Welt-2\n" +
		"30,3.03,false,Welt-3\n" +
		"40,4.04,true,Welt-4\n" +
		"50,5.05,false,Welt-5\n" +
		"60,6.06,true,Welt-6\n" +
		"70,7.07,false,Welt-7\n" +
		"80,8.08,true,Welt-8\n";

	TestBaseUtils.compareResultAsText(results, expected);
}
 

/**
 * Validates the use of Pravega Table Descriptor to generate the source/sink Table factory to
 * write and read from Pravega stream using {@link BatchTableEnvironment}
 * @throws Exception
 */
@Test
public void testBatchTableUsingDescriptor() throws Exception {

    final String scope = setupUtils.getScope();
    final String streamName = "stream";
    Stream stream = Stream.of(scope, streamName);
    this.setupUtils.createTestStream(stream.getStreamName(), 1);

    ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
    env.setParallelism(1);
    BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env);

    PravegaConfig pravegaConfig = setupUtils.getPravegaConfig();

    Pravega pravega = new Pravega();
    pravega.tableSinkWriterBuilder()
            .withRoutingKeyField("category")
            .forStream(stream)
            .withPravegaConfig(pravegaConfig);
    pravega.tableSourceReaderBuilder()
            .withReaderGroupScope(stream.getScope())
            .forStream(stream)
            .withPravegaConfig(pravegaConfig);

    ConnectTableDescriptor desc = tableEnv.connect(pravega)
            .withFormat(new Json().failOnMissingField(false))
            .withSchema(new Schema().
                    field("category", DataTypes.STRING()).
                    field("value", DataTypes.INT()));
    desc.createTemporaryTable("test");

    final Map<String, String> propertiesMap = desc.toProperties();
    final TableSink<?> sink = TableFactoryService.find(BatchTableSinkFactory.class, propertiesMap)
            .createBatchTableSink(propertiesMap);
    final TableSource<?> source = TableFactoryService.find(BatchTableSourceFactory.class, propertiesMap)
            .createBatchTableSource(propertiesMap);

    Table table = tableEnv.fromDataSet(env.fromCollection(SAMPLES));

    String tableSinkPath = tableEnv.getCurrentDatabase() + "." + "PravegaSink";

    ConnectorCatalogTable<?, ?> connectorCatalogTableSink = ConnectorCatalogTable.sink(sink, true);

    tableEnv.getCatalog(tableEnv.getCurrentCatalog()).get().createTable(
            ObjectPath.fromString(tableSinkPath),
            connectorCatalogTableSink, false);

    table.insertInto("PravegaSink");
    env.execute();

    String tableSourcePath = tableEnv.getCurrentDatabase() + "." + "samples";

    ConnectorCatalogTable<?, ?> connectorCatalogTableSource = ConnectorCatalogTable.source(source, true);

    tableEnv.getCatalog(tableEnv.getCurrentCatalog()).get().createTable(
            ObjectPath.fromString(tableSourcePath),
            connectorCatalogTableSource, false);

    // select some sample data from the Pravega-backed table, as a view
    Table view = tableEnv.sqlQuery("SELECT * FROM samples WHERE category IN ('A','B')");

    // convert the view to a dataset and collect the results for comparison purposes
    List<SampleRecord> results = tableEnv.toDataSet(view, SampleRecord.class).collect();
    Assert.assertEquals(new HashSet<>(SAMPLES), new HashSet<>(results));
}
 

public static void main(String[] args) {
    //流作业
    StreamTableEnvironment sEnv = StreamTableEnvironment.create(StreamExecutionEnvironment.getExecutionEnvironment());

    Catalog catalog = new GenericInMemoryCatalog("zhisheng");
    sEnv.registerCatalog("InMemCatalog", catalog);


    //批作业
    BatchTableEnvironment bEnv = BatchTableEnvironment.create(ExecutionEnvironment.getExecutionEnvironment());

}