org.apache.flink.table.factories.BatchTableSourceFactory Java Examples

The following examples show how to use org.apache.flink.table.factories.BatchTableSourceFactory. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: FlinkPravegaTableITCase.java    From flink-connectors with Apache License 2.0 4 votes vote down vote up 
private void testTableSourceBatchDescriptor(Stream stream, PravegaConfig pravegaConfig) throws Exception {
    ExecutionEnvironment execEnvRead = ExecutionEnvironment.getExecutionEnvironment();
    // Can only use Legacy Flink planner for BatchTableEnvironment
    BatchTableEnvironment tableEnv = BatchTableEnvironment.create(execEnvRead);
    execEnvRead.setParallelism(1);

    Schema schema = new Schema()
            .field("user", DataTypes.STRING())
            .field("uri", DataTypes.STRING())
            // Note: LocalDateTime is not supported in legacy Flink planner, bridged to Timestamp with the data source.
            // See https://issues.apache.org/jira/browse/FLINK-16693 for more information.
            .field("accessTime", DataTypes.TIMESTAMP(3).bridgedTo(Timestamp.class));

    Pravega pravega = new Pravega();

    pravega.tableSourceReaderBuilder()
            .withReaderGroupScope(stream.getScope())
            .forStream(stream)
            .withPravegaConfig(pravegaConfig);

    ConnectTableDescriptor desc = tableEnv.connect(pravega)
            .withFormat(new Json().failOnMissingField(false))
            .withSchema(schema);

    final Map<String, String> propertiesMap = desc.toProperties();
    final TableSource<?> source = TableFactoryService.find(BatchTableSourceFactory.class, propertiesMap)
            .createBatchTableSource(propertiesMap);

    String tableSourcePath = tableEnv.getCurrentDatabase() + "." + "MyTableRow";

    ConnectorCatalogTable<?, ?> connectorCatalogSourceTable = ConnectorCatalogTable.source(source, true);

    tableEnv.getCatalog(tableEnv.getCurrentCatalog()).get().createTable(
            ObjectPath.fromString(tableSourcePath),
            connectorCatalogSourceTable, false);

    String sqlQuery = "SELECT user, count(uri) from MyTableRow GROUP BY user";

    Table result = tableEnv.sqlQuery(sqlQuery);

    DataSet<Row> resultSet = tableEnv.toDataSet(result, Row.class);

    List<Row> results = resultSet.collect();
    log.info("results: {}", results);

    boolean compare = compare(results, getExpectedResultsRetracted());
    assertTrue("Output does not match expected result", compare);
}
Example #2
Source File: FlinkTableITCase.java    From flink-connectors with Apache License 2.0 4 votes vote down vote up 
/**
 * Validates the use of Pravega Table Descriptor to generate the source/sink Table factory to
 * write and read from Pravega stream using {@link BatchTableEnvironment}
 * @throws Exception
 */
@Test
public void testBatchTableUsingDescriptor() throws Exception {

    final String scope = setupUtils.getScope();
    final String streamName = "stream";
    Stream stream = Stream.of(scope, streamName);
    this.setupUtils.createTestStream(stream.getStreamName(), 1);

    ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
    env.setParallelism(1);
    BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env);

    PravegaConfig pravegaConfig = setupUtils.getPravegaConfig();

    Pravega pravega = new Pravega();
    pravega.tableSinkWriterBuilder()
            .withRoutingKeyField("category")
            .forStream(stream)
            .withPravegaConfig(pravegaConfig);
    pravega.tableSourceReaderBuilder()
            .withReaderGroupScope(stream.getScope())
            .forStream(stream)
            .withPravegaConfig(pravegaConfig);

    ConnectTableDescriptor desc = tableEnv.connect(pravega)
            .withFormat(new Json().failOnMissingField(false))
            .withSchema(new Schema().
                    field("category", DataTypes.STRING()).
                    field("value", DataTypes.INT()));
    desc.createTemporaryTable("test");

    final Map<String, String> propertiesMap = desc.toProperties();
    final TableSink<?> sink = TableFactoryService.find(BatchTableSinkFactory.class, propertiesMap)
            .createBatchTableSink(propertiesMap);
    final TableSource<?> source = TableFactoryService.find(BatchTableSourceFactory.class, propertiesMap)
            .createBatchTableSource(propertiesMap);

    Table table = tableEnv.fromDataSet(env.fromCollection(SAMPLES));

    String tableSinkPath = tableEnv.getCurrentDatabase() + "." + "PravegaSink";

    ConnectorCatalogTable<?, ?> connectorCatalogTableSink = ConnectorCatalogTable.sink(sink, true);

    tableEnv.getCatalog(tableEnv.getCurrentCatalog()).get().createTable(
            ObjectPath.fromString(tableSinkPath),
            connectorCatalogTableSink, false);

    table.insertInto("PravegaSink");
    env.execute();

    String tableSourcePath = tableEnv.getCurrentDatabase() + "." + "samples";

    ConnectorCatalogTable<?, ?> connectorCatalogTableSource = ConnectorCatalogTable.source(source, true);

    tableEnv.getCatalog(tableEnv.getCurrentCatalog()).get().createTable(
            ObjectPath.fromString(tableSourcePath),
            connectorCatalogTableSource, false);

    // select some sample data from the Pravega-backed table, as a view
    Table view = tableEnv.sqlQuery("SELECT * FROM samples WHERE category IN ('A','B')");

    // convert the view to a dataset and collect the results for comparison purposes
    List<SampleRecord> results = tableEnv.toDataSet(view, SampleRecord.class).collect();
    Assert.assertEquals(new HashSet<>(SAMPLES), new HashSet<>(results));
}