Java Code Examples for org.apache.flink.table.api.java.BatchTableEnvironment#fromDataSet()

@Test
public void testAsFromTupleByName() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env, config());

	Table table = tableEnv.fromDataSet(CollectionDataSets.get3TupleDataSet(env), "f2");

	DataSet<Row> ds = tableEnv.toDataSet(table, Row.class);
	List<Row> results = ds.collect();
	String expected = "Hi\n" + "Hello\n" + "Hello world\n" +
		"Hello world, how are you?\n" + "I am fine.\n" + "Luke Skywalker\n" +
		"Comment#1\n" + "Comment#2\n" + "Comment#3\n" + "Comment#4\n" +
		"Comment#5\n" + "Comment#6\n" + "Comment#7\n" +
		"Comment#8\n" + "Comment#9\n" + "Comment#10\n" +
		"Comment#11\n" + "Comment#12\n" + "Comment#13\n" +
		"Comment#14\n" + "Comment#15\n";
	compareResultAsText(results, expected);
}
 

public static void main(String[] args) throws Exception {
    ExecutionEnvironment env = ExecutionEnvironment.createCollectionsEnvironment();
    BatchTableEnvironment tEnv = BatchTableEnvironment.create(env);

    DataSet<WC> input = env.fromElements(
            new WC("Hello", 1),
            new WC("zhisheng", 2),
            new WC("Hello", 1));

    Table table = tEnv.fromDataSet(input);

    Table filtered = table
            .groupBy("word")
            .select("word, c.sum as c")
            .filter("c = 2");

    DataSet<WC> result = tEnv.toDataSet(filtered, WC.class);

    result.print();
}
 

public static void main(String[] args) throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.createCollectionsEnvironment();
	BatchTableEnvironment tEnv = BatchTableEnvironment.create(env);

	DataSet<WC> input = env.fromElements(
			new WC("Hello", 1),
			new WC("Ciao", 1),
			new WC("Hello", 1));

	Table table = tEnv.fromDataSet(input);

	Table filtered = table
			.groupBy("word")
			.select("word, frequency.sum as frequency")
			.filter("frequency = 2");

	DataSet<WC> result = tEnv.toDataSet(filtered, WC.class);

	result.print();
}
 

@Test
public void testTableRegister() throws Exception {
	final String tableName = "MyTable";
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env, config());

	DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
	Table t = tableEnv.fromDataSet(ds);
	tableEnv.registerTable(tableName, t);
	Table result = tableEnv.scan(tableName).select("f0, f1").filter("f0 > 7");

	DataSet<Row> resultSet = tableEnv.toDataSet(result, Row.class);
	List<Row> results = resultSet.collect();
	String expected = "8,4\n" + "9,4\n" + "10,4\n" + "11,5\n" + "12,5\n" +
			"13,5\n" + "14,5\n" + "15,5\n" +
			"16,6\n" + "17,6\n" + "18,6\n" + "19,6\n" + "20,6\n" + "21,6\n";
	compareResultAsText(results, expected);
}
 

@Test(expected = ValidationException.class)
public void testIllegalWhitespaceOnlyName() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env, config());

	DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
	Table t = tableEnv.fromDataSet(ds);
	// Must fail. Table is empty
	tableEnv.registerTable("     ", t);
}
 

@Test(expected = ValidationException.class)
public void testGenericRowWithAlias() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env, config());

	// use null value the enforce GenericType
	DataSet<Row> dataSet = env.fromElements(Row.of((Integer) null));
	assertTrue(dataSet.getType() instanceof GenericTypeInfo);
	assertTrue(dataSet.getType().getTypeClass().equals(Row.class));

	// Must fail. Cannot import DataSet<Row> with GenericTypeInfo.
	tableEnv.fromDataSet(dataSet, "nullField");
}
 

@Test(expected = TableException.class)
public void testCustomCalciteConfig() {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env, config());

	PlannerConfig cc = new CalciteConfigBuilder()
			.replaceLogicalOptRuleSet(RuleSets.ofList())
			.replacePhysicalOptRuleSet(RuleSets.ofList())
			.build();
	tableEnv.getConfig().setPlannerConfig(cc);

	DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
	Table t = tableEnv.fromDataSet(ds);
	tableEnv.toDataSet(t, Row.class);
}
 

@Test
public void test() throws Exception {
    BatchTableEnvironment environment = MLEnvironmentFactory.getDefault().getBatchTableEnvironment();
    DataSet<Row> input = MLEnvironmentFactory.getDefault().getExecutionEnvironment().fromElements(Row.of("a b c d a b c"));
    Table table = environment.fromDataSet(input);
    environment.registerFunction("DocWordSplitCount", new DocWordSplitCount(" "));
    environment.registerTable("myTable", table);
    List<Row> list = environment.toDataSet(
        environment.sqlQuery("SELECT w, cnt FROM myTable, LATERAL TABLE(DocWordSplitCount(f0)) as T(w, cnt)"),
        Row.class).collect();
    Assert.assertArrayEquals(list.toArray(),
        new Row[] {Row.of("a", 2L), Row.of("b", 2L), Row.of("c", 2L), Row.of("d", 1L)});
}
 

@Test(expected = TableException.class)
public void testAsWithAmbiguousFields() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env, config());

	// Must fail. Specified field names are not unique.
	tableEnv.fromDataSet(CollectionDataSets.get3TupleDataSet(env), "a, b, b");
}
 

@Test(expected = TableException.class)
public void testAsWithNonFieldReference1() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env, config());

	// Must fail. as() does only allow field name expressions
	tableEnv.fromDataSet(CollectionDataSets.get3TupleDataSet(env), "a + 1, b, c");
}
 

@Test
public void testAvroToAvro() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tEnv = BatchTableEnvironment.create(env, config());

	Table t = tEnv.fromDataSet(testData(env));
	Table result = t.select("*");

	List<User> results = tEnv.toDataSet(result, Types.POJO(User.class)).collect();
	List<User> expected = Arrays.asList(USER_1, USER_2, USER_3);
	assertEquals(expected, results);
}
 

@Test(expected = ValidationException.class)
public void testIllegalEmptyName() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env, config());

	DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
	Table t = tableEnv.fromDataSet(ds);
	// Must fail. Table is empty
	tableEnv.registerTable("", t);
}
 

@Test(expected = TableException.class)
public void testNonStaticClassOutput() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env, config());

	// Must fail since class is not static
	Table t = tableEnv.fromDataSet(env.fromElements(1, 2, 3), "number");
	tableEnv.toDataSet(t, MyNonStatic.class);
}
 

@Test
public void testBatchTableSinkUsingDescriptor() throws Exception {

    // create a Pravega stream for test purposes
    Stream stream = Stream.of(setupUtils.getScope(), "testBatchTableSinkUsingDescriptor");
    this.setupUtils.createTestStream(stream.getStreamName(), 1);

    // create a Flink Table environment
    ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
    env.setParallelism(1);
    BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env);

    Table table = tableEnv.fromDataSet(env.fromCollection(SAMPLES));

    Pravega pravega = new Pravega();
    pravega.tableSinkWriterBuilder()
            .withRoutingKeyField("category")
            .forStream(stream)
            .withPravegaConfig(setupUtils.getPravegaConfig());

    ConnectTableDescriptor desc = tableEnv.connect(pravega)
            .withFormat(new Json().failOnMissingField(true))
            .withSchema(new Schema().field("category", DataTypes.STRING()).
                    field("value", DataTypes.INT()));
    desc.createTemporaryTable("test");

    final Map<String, String> propertiesMap = desc.toProperties();
    final TableSink<?> sink = TableFactoryService.find(BatchTableSinkFactory.class, propertiesMap)
            .createBatchTableSink(propertiesMap);

    String tableSinkPath = tableEnv.getCurrentDatabase() + "." + "PravegaSink";

    ConnectorCatalogTable<?, ?> connectorCatalogSinkTable = ConnectorCatalogTable.sink(sink, true);

    tableEnv.getCatalog(tableEnv.getCurrentCatalog()).get().createTable(
            ObjectPath.fromString(tableSinkPath),
            connectorCatalogSinkTable, false);
    table.insertInto("PravegaSink");
    env.execute();
}
 

@Test
public void testAvroToRow() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.getConfig().registerTypeWithKryoSerializer(LocalDate.class, AvroKryoSerializerUtils.JodaLocalDateSerializer.class);
	env.getConfig().registerTypeWithKryoSerializer(LocalTime.class, AvroKryoSerializerUtils.JodaLocalTimeSerializer.class);
	BatchTableEnvironment tEnv = BatchTableEnvironment.create(env, config());

	Table t = tEnv.fromDataSet(testData(env));
	Table result = t.select("*");

	List<Row> results = tEnv.toDataSet(result, Row.class).collect();
	String expected =
		"black,null,Whatever,[true],[hello],true,java.nio.HeapByteBuffer[pos=0 lim=10 cap=10]," +
		"2014-03-01,java.nio.HeapByteBuffer[pos=0 lim=2 cap=2],[7, -48],0.0,GREEN," +
		"[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],42,{},null,null,null,123456," +
		"12:12:12.000,123456,2014-03-01T12:12:12.321Z,null\n" +
		"blue,null,Charlie,[],[],false,java.nio.HeapByteBuffer[pos=0 lim=10 cap=10],2014-03-01," +
		"java.nio.HeapByteBuffer[pos=0 lim=2 cap=2],[7, -48],1.337,RED,null,1337,{}," +
		"{\"num\": 42, \"street\": \"Bakerstreet\", \"city\": \"Berlin\", \"state\": " +
		"\"Berlin\", \"zip\": \"12049\"},null,null,123456,12:12:12.000,123456," +
		"2014-03-01T12:12:12.321Z,null\n" +
		"yellow,null,Terminator,[false],[world],false," +
		"java.nio.HeapByteBuffer[pos=0 lim=10 cap=10],2014-03-01," +
		"java.nio.HeapByteBuffer[pos=0 lim=2 cap=2],[7, -48],0.0,GREEN," +
		"[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],1,{},null,null,null,123456," +
		"12:12:12.000,123456,2014-03-01T12:12:12.321Z,null";
	TestBaseUtils.compareResultAsText(results, expected);
}
 

@Test
public void testAvroStringAccess() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tEnv = BatchTableEnvironment.create(env, config());

	Table t = tEnv.fromDataSet(testData(env));
	Table result = t.select("name");
	List<Utf8> results = tEnv.toDataSet(result, Types.GENERIC(Utf8.class)).collect();
	String expected = "Charlie\n" +
			"Terminator\n" +
			"Whatever";
	TestBaseUtils.compareResultAsText(results, expected);
}
 

@Test
public void testAvroObjectAccess() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tEnv = BatchTableEnvironment.create(env, config());

	Table t = tEnv.fromDataSet(testData(env));
	Table result = t
			.filter("type_nested.isNotNull")
			.select("type_nested.flatten()").as("city, num, state, street, zip");

	List<Address> results = tEnv.toDataSet(result, Types.POJO(Address.class)).collect();
	String expected = USER_1.getTypeNested().toString();
	TestBaseUtils.compareResultAsText(results, expected);
}
 

@Test
public void testAvroToAvro() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tEnv = BatchTableEnvironment.create(env, config());

	Table t = tEnv.fromDataSet(testData(env));
	Table result = t.select("*");

	List<User> results = tEnv.toDataSet(result, Types.POJO(User.class)).collect();
	List<User> expected = Arrays.asList(USER_1, USER_2, USER_3);
	assertEquals(expected, results);
}
 

@Test(expected = ValidationException.class)
public void testAsWithTooManyFields() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env, config());

	// Must fail. Too many field names specified.
	tableEnv.fromDataSet(CollectionDataSets.get3TupleDataSet(env), "a, b, c, d");
}
 

@Test(expected = ValidationException.class)
public void testNonStaticClassInput() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env, config());

	// Must fail since class is not static
	tableEnv.fromDataSet(env.fromElements(new MyNonStatic()), "name");
}