org.apache.flink.table.catalog.ConnectorCatalogTable Java Examples

private void registerTableSinkInternal(String name, TableSink<?> tableSink) {
	Optional<CatalogBaseTable> table = getCatalogTable(catalogManager.getBuiltInCatalogName(),
		catalogManager.getBuiltInDatabaseName(), name);

	if (table.isPresent()) {
		if (table.get() instanceof ConnectorCatalogTable<?, ?>) {
			ConnectorCatalogTable<?, ?> sourceSinkTable = (ConnectorCatalogTable<?, ?>) table.get();
			if (sourceSinkTable.getTableSink().isPresent()) {
				throw new ValidationException(String.format(
					"Table '%s' already exists. Please choose a different name.", name));
			} else {
				// wrapper contains only sink (not source)
				replaceTableInternal(
					name,
					ConnectorCatalogTable
						.sourceAndSink(sourceSinkTable.getTableSource().get(), tableSink, !IS_STREAM_TABLE));
			}
		} else {
			throw new ValidationException(String.format(
				"Table '%s' already exists. Please choose a different name.", name));
		}
	} else {
		registerTableInternal(name, ConnectorCatalogTable.sink(tableSink, !IS_STREAM_TABLE));
	}
}
 

@Test
public void testGetFlinkPreparingTableBase() {
	// Mock CatalogSchemaTable.
	TableSchema schema = TableSchema.builder().build();
	CatalogSchemaTable mockTable = new CatalogSchemaTable(
		ObjectIdentifier.of("a", "b", "c"),
		CatalogManager.TableLookupResult.permanent(ConnectorCatalogTable.source(
			new TestTableSource(true, schema),
			true), schema),
		FlinkStatistic.UNKNOWN(),
		null,
		true);

	rootSchemaPlus.add(tableMockName, mockTable);
	Prepare.PreparingTable preparingTable = catalogReader
		.getTable(Collections.singletonList(tableMockName));
	assertTrue(preparingTable instanceof FlinkPreparingTableBase);
}
 

@Test
public void testGetStatsFromCatalogForConnectorCatalogTable() throws Exception {
	catalog.createTable(
			new ObjectPath(databaseName, "T1"),
			ConnectorCatalogTable.source(new TestTableSource(true, tableSchema), true),
			false);
	catalog.createTable(
			new ObjectPath(databaseName, "T2"),
			ConnectorCatalogTable.source(new TestTableSource(true, tableSchema), true),
			false);

	alterTableStatistics(catalog, "T1");
	assertStatistics(tEnv, "T1");

	alterTableStatisticsWithUnknownRowCount(catalog, "T2");
	assertTableStatisticsWithUnknownRowCount(tEnv, "T2");
}
 

private void registerTableSourceInternal(String name, TableSource<?> tableSource) {
	validateTableSource(tableSource);
	Optional<CatalogBaseTable> table = getCatalogTable(catalogManager.getBuiltInCatalogName(),
		catalogManager.getBuiltInDatabaseName(), name);

	if (table.isPresent()) {
		if (table.get() instanceof ConnectorCatalogTable<?, ?>) {
			ConnectorCatalogTable<?, ?> sourceSinkTable = (ConnectorCatalogTable<?, ?>) table.get();
			if (sourceSinkTable.getTableSource().isPresent()) {
				throw new ValidationException(String.format(
					"Table '%s' already exists. Please choose a different name.", name));
			} else {
				// wrapper contains only sink (not source)
				replaceTableInternal(
					name,
					ConnectorCatalogTable
						.sourceAndSink(tableSource, sourceSinkTable.getTableSink().get(), !IS_STREAM_TABLE));
			}
		} else {
			throw new ValidationException(String.format(
				"Table '%s' already exists. Please choose a different name.", name));
		}
	} else {
		registerTableInternal(name, ConnectorCatalogTable.source(tableSource, !IS_STREAM_TABLE));
	}
}
 

@Test
public void testBatchTableSinkUsingDescriptor() throws Exception {

    // create a Pravega stream for test purposes
    Stream stream = Stream.of(setupUtils.getScope(), "testBatchTableSinkUsingDescriptor");
    this.setupUtils.createTestStream(stream.getStreamName(), 1);

    // create a Flink Table environment
    ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
    env.setParallelism(1);
    BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env);

    Table table = tableEnv.fromDataSet(env.fromCollection(SAMPLES));

    Pravega pravega = new Pravega();
    pravega.tableSinkWriterBuilder()
            .withRoutingKeyField("category")
            .forStream(stream)
            .withPravegaConfig(setupUtils.getPravegaConfig());

    ConnectTableDescriptor desc = tableEnv.connect(pravega)
            .withFormat(new Json().failOnMissingField(true))
            .withSchema(new Schema().field("category", DataTypes.STRING()).
                    field("value", DataTypes.INT()));
    desc.createTemporaryTable("test");

    final Map<String, String> propertiesMap = desc.toProperties();
    final TableSink<?> sink = TableFactoryService.find(BatchTableSinkFactory.class, propertiesMap)
            .createBatchTableSink(propertiesMap);

    String tableSinkPath = tableEnv.getCurrentDatabase() + "." + "PravegaSink";

    ConnectorCatalogTable<?, ?> connectorCatalogSinkTable = ConnectorCatalogTable.sink(sink, true);

    tableEnv.getCatalog(tableEnv.getCurrentCatalog()).get().createTable(
            ObjectPath.fromString(tableSinkPath),
            connectorCatalogSinkTable, false);
    table.insertInto("PravegaSink");
    env.execute();
}
 

private static FlinkPreparingTableBase convertSourceTable(
		RelOptSchema relOptSchema,
		RelDataType rowType,
		ObjectIdentifier tableIdentifier,
		ConnectorCatalogTable<?, ?> table,
		FlinkStatistic statistic,
		boolean isStreamingMode) {
	TableSource<?> tableSource = table.getTableSource().get();
	if (!(tableSource instanceof StreamTableSource ||
		tableSource instanceof LookupableTableSource)) {
		throw new ValidationException(
			"Only StreamTableSource and LookupableTableSource can be used in Blink planner.");
	}
	if (!isStreamingMode && tableSource instanceof StreamTableSource &&
		!((StreamTableSource<?>) tableSource).isBounded()) {
		throw new ValidationException("Only bounded StreamTableSource can be used in batch mode.");
	}

	return new LegacyTableSourceTable<>(
		relOptSchema,
		tableIdentifier,
		rowType,
		statistic,
		tableSource,
		isStreamingMode,
		table);
}
 

@Override
public void registerTableSinkInternal(String name, TableSink<?> tableSink) {
	ObjectIdentifier objectIdentifier = catalogManager.qualifyIdentifier(UnresolvedIdentifier.of(name));
	Optional<CatalogBaseTable> table = getTemporaryTable(objectIdentifier);

	if (table.isPresent()) {
		if (table.get() instanceof ConnectorCatalogTable<?, ?>) {
			ConnectorCatalogTable<?, ?> sourceSinkTable = (ConnectorCatalogTable<?, ?>) table.get();
			if (sourceSinkTable.getTableSink().isPresent()) {
				throw new ValidationException(String.format(
					"Table '%s' already exists. Please choose a different name.", name));
			} else {
				// wrapper contains only sink (not source)
				ConnectorCatalogTable sourceAndSink = ConnectorCatalogTable
					.sourceAndSink(sourceSinkTable.getTableSource().get(), tableSink, !IS_STREAM_TABLE);
				catalogManager.dropTemporaryTable(objectIdentifier, false);
				catalogManager.createTemporaryTable(sourceAndSink, objectIdentifier, false);
			}
		} else {
			throw new ValidationException(String.format(
				"Table '%s' already exists. Please choose a different name.", name));
		}
	} else {
		ConnectorCatalogTable sink = ConnectorCatalogTable.sink(tableSink, !IS_STREAM_TABLE);
		catalogManager.createTemporaryTable(sink, objectIdentifier, false);
	}
}
 

@Override
public void registerTableSourceInternal(String name, TableSource<?> tableSource) {
	validateTableSource(tableSource);
	ObjectIdentifier objectIdentifier = catalogManager.qualifyIdentifier(UnresolvedIdentifier.of(name));
	Optional<CatalogBaseTable> table = getTemporaryTable(objectIdentifier);

	if (table.isPresent()) {
		if (table.get() instanceof ConnectorCatalogTable<?, ?>) {
			ConnectorCatalogTable<?, ?> sourceSinkTable = (ConnectorCatalogTable<?, ?>) table.get();
			if (sourceSinkTable.getTableSource().isPresent()) {
				throw new ValidationException(String.format(
					"Table '%s' already exists. Please choose a different name.", name));
			} else {
				// wrapper contains only sink (not source)
				ConnectorCatalogTable sourceAndSink = ConnectorCatalogTable.sourceAndSink(
					tableSource,
					sourceSinkTable.getTableSink().get(),
					!IS_STREAM_TABLE);
				catalogManager.dropTemporaryTable(objectIdentifier, false);
				catalogManager.createTemporaryTable(sourceAndSink, objectIdentifier, false);
			}
		} else {
			throw new ValidationException(String.format(
				"Table '%s' already exists. Please choose a different name.", name));
		}
	} else {
		ConnectorCatalogTable source = ConnectorCatalogTable.source(tableSource, !IS_STREAM_TABLE);
		catalogManager.createTemporaryTable(source, objectIdentifier, false);
	}
}
 

private Table convertConnectorTable(
		ConnectorCatalogTable<?, ?> table,
		ObjectPath tablePath) throws TableNotExistException {
	if (table.getTableSource().isPresent()) {
		TableSource<?> tableSource = table.getTableSource().get();
		if (!(tableSource instanceof StreamTableSource ||
				tableSource instanceof LookupableTableSource)) {
			throw new TableException(
					"Only StreamTableSource and LookupableTableSource can be used in Blink planner.");
		}
		if (!isStreamingMode && tableSource instanceof StreamTableSource &&
				!((StreamTableSource<?>) tableSource).isBounded()) {
			throw new TableException("Only bounded StreamTableSource can be used in batch mode.");
		}

		TableStats tableStats = TableStats.UNKNOWN;
		// TODO supports stats for partitionable table
		if (!table.isPartitioned()) {
			CatalogTableStatistics tableStatistics = catalog.getTableStatistics(tablePath);
			CatalogColumnStatistics columnStatistics = catalog.getTableColumnStatistics(tablePath);
			tableStats = convertToTableStats(tableStatistics, columnStatistics);
		}
		return new TableSourceTable<>(
				tableSource,
				isStreamingMode,
				FlinkStatistic.builder().tableStats(tableStats).build());
	} else {
		Optional<TableSinkTable> tableSinkTable = table.getTableSink()
			.map(tableSink -> new TableSinkTable<>(
				tableSink,
				FlinkStatistic.UNKNOWN()));
		if (tableSinkTable.isPresent()) {
			return tableSinkTable.get();
		} else {
			throw new TableException("Cannot convert a connector table " +
				"without either source or sink.");
		}
	}
}
 

@Test
public void testConnect() throws Exception {
	final TableEnvironmentMock tableEnv = TableEnvironmentMock.getStreamingInstance();

	tableEnv
		.connect(new ConnectorDescriptorMock(TableSourceFactoryMock.CONNECTOR_TYPE_VALUE, 1, true))
		.withFormat(new FormatDescriptorMock("my_format", 1))
		.withSchema(new Schema()
			.field("my_field_0", "INT")
			.field("my_field_1", "BOOLEAN"))
		.inAppendMode()
		.registerTableSource("my_table");

	final Catalog catalog = tableEnv.getCatalog(EnvironmentSettings.DEFAULT_BUILTIN_CATALOG)
		.orElseThrow(AssertionError::new);

	final CatalogBaseTable table = catalog
		.getTable(new ObjectPath(EnvironmentSettings.DEFAULT_BUILTIN_DATABASE, "my_table"));

	assertThat(
		table.getSchema(),
		equalTo(
			TableSchema.builder()
				.field("my_field_0", DataTypes.INT())
				.field("my_field_1", DataTypes.BOOLEAN())
				.build()));

	final ConnectorCatalogTable<?, ?> connectorCatalogTable = (ConnectorCatalogTable<?, ?>) table;

	assertThat(
		connectorCatalogTable.getTableSource().isPresent(),
		equalTo(true));
}
 

/**
 * Validates the use of Pravega Table Descriptor to generate the source/sink Table factory to
 * write and read from Pravega stream using {@link BatchTableEnvironment}
 * @throws Exception
 */
@Test
public void testBatchTableUsingDescriptor() throws Exception {

    final String scope = setupUtils.getScope();
    final String streamName = "stream";
    Stream stream = Stream.of(scope, streamName);
    this.setupUtils.createTestStream(stream.getStreamName(), 1);

    ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
    env.setParallelism(1);
    BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env);

    PravegaConfig pravegaConfig = setupUtils.getPravegaConfig();

    Pravega pravega = new Pravega();
    pravega.tableSinkWriterBuilder()
            .withRoutingKeyField("category")
            .forStream(stream)
            .withPravegaConfig(pravegaConfig);
    pravega.tableSourceReaderBuilder()
            .withReaderGroupScope(stream.getScope())
            .forStream(stream)
            .withPravegaConfig(pravegaConfig);

    ConnectTableDescriptor desc = tableEnv.connect(pravega)
            .withFormat(new Json().failOnMissingField(false))
            .withSchema(new Schema().
                    field("category", DataTypes.STRING()).
                    field("value", DataTypes.INT()));
    desc.createTemporaryTable("test");

    final Map<String, String> propertiesMap = desc.toProperties();
    final TableSink<?> sink = TableFactoryService.find(BatchTableSinkFactory.class, propertiesMap)
            .createBatchTableSink(propertiesMap);
    final TableSource<?> source = TableFactoryService.find(BatchTableSourceFactory.class, propertiesMap)
            .createBatchTableSource(propertiesMap);

    Table table = tableEnv.fromDataSet(env.fromCollection(SAMPLES));

    String tableSinkPath = tableEnv.getCurrentDatabase() + "." + "PravegaSink";

    ConnectorCatalogTable<?, ?> connectorCatalogTableSink = ConnectorCatalogTable.sink(sink, true);

    tableEnv.getCatalog(tableEnv.getCurrentCatalog()).get().createTable(
            ObjectPath.fromString(tableSinkPath),
            connectorCatalogTableSink, false);

    table.insertInto("PravegaSink");
    env.execute();

    String tableSourcePath = tableEnv.getCurrentDatabase() + "." + "samples";

    ConnectorCatalogTable<?, ?> connectorCatalogTableSource = ConnectorCatalogTable.source(source, true);

    tableEnv.getCatalog(tableEnv.getCurrentCatalog()).get().createTable(
            ObjectPath.fromString(tableSourcePath),
            connectorCatalogTableSource, false);

    // select some sample data from the Pravega-backed table, as a view
    Table view = tableEnv.sqlQuery("SELECT * FROM samples WHERE category IN ('A','B')");

    // convert the view to a dataset and collect the results for comparison purposes
    List<SampleRecord> results = tableEnv.toDataSet(view, SampleRecord.class).collect();
    Assert.assertEquals(new HashSet<>(SAMPLES), new HashSet<>(results));
}
 

@Test
public void testStreamTableSinkUsingDescriptor() throws Exception {

    // create a Pravega stream for test purposes
    Stream stream = Stream.of(setupUtils.getScope(), "testStreamTableSinkUsingDescriptor");
    this.setupUtils.createTestStream(stream.getStreamName(), 1);

    // create a Flink Table environment
    StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment().setParallelism(1);
    StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env,
            EnvironmentSettings.newInstance()
                    // watermark is only supported in blink planner
                    .useBlinkPlanner()
                    .inStreamingMode()
                    .build());

    Table table = tableEnv.fromDataStream(env.fromCollection(SAMPLES));

    Pravega pravega = new Pravega();
    pravega.tableSinkWriterBuilder()
            .withRoutingKeyField("category")
            .forStream(stream)
            .withPravegaConfig(setupUtils.getPravegaConfig());

    ConnectTableDescriptor desc = tableEnv.connect(pravega)
            .withFormat(new Json().failOnMissingField(true))
            .withSchema(new Schema().
                    field("category", DataTypes.STRING())
                    .field("value", DataTypes.INT()))
            .inAppendMode();
    desc.createTemporaryTable("test");

    final Map<String, String> propertiesMap = desc.toProperties();
    final TableSink<?> sink = TableFactoryService.find(StreamTableSinkFactory.class, propertiesMap)
            .createStreamTableSink(propertiesMap);

    String tablePath = tableEnv.getCurrentDatabase() + "." + "PravegaSink";

    ConnectorCatalogTable<?, ?> connectorCatalogTable = ConnectorCatalogTable.sink(sink, false);

    tableEnv.getCatalog(tableEnv.getCurrentCatalog()).get().createTable(
            ObjectPath.fromString(tablePath),
            connectorCatalogTable, false);

    table.insertInto("PravegaSink");
    env.execute();
}
 

@Test
public void testStreamTableSinkUsingDescriptorWithWatermark() throws Exception {
    // create a Pravega stream for test purposes
    Stream stream = Stream.of(setupUtils.getScope(), "testStreamTableSinkUsingDescriptorWithWatermark");
    this.setupUtils.createTestStream(stream.getStreamName(), 1);

    // create a Flink Table environment
    StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment().setParallelism(1);
    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
    StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env,
            EnvironmentSettings.newInstance()
                    // watermark is only supported in blink planner
                    .useBlinkPlanner()
                    .inStreamingMode()
                    .build());
    DataStream<SampleRecordWithTimestamp> dataStream = env.fromCollection(SAMPLES)
            .map(SampleRecordWithTimestamp::new)
            .assignTimestampsAndWatermarks(new AscendingTimestampExtractor<SampleRecordWithTimestamp>() {
                @Override
                public long extractAscendingTimestamp(SampleRecordWithTimestamp sampleRecordWithTimestamp) {
                    return sampleRecordWithTimestamp.getTimestamp();
                }
            });

    Table table = tableEnv.fromDataStream(dataStream, "category, value, UserActionTime.rowtime");

    Pravega pravega = new Pravega();
    pravega.tableSinkWriterBuilder()
            .withRoutingKeyField("category")
            .enableWatermark(true)
            .forStream(stream)
            .withPravegaConfig(setupUtils.getPravegaConfig());

    TableSchema tableSchema = TableSchema.builder()
            .field("category", DataTypes.STRING())
            .field("value", DataTypes.INT())
            .field("timestamp", DataTypes.TIMESTAMP(3))
            .build();

    Schema schema = new Schema().schema(tableSchema);

    ConnectTableDescriptor desc = tableEnv.connect(pravega)
            .withFormat(new Json().failOnMissingField(true))
            .withSchema(schema)
            .inAppendMode();
    desc.createTemporaryTable("test");

    final Map<String, String> propertiesMap = desc.toProperties();
    final TableSink<?> sink = TableFactoryService.find(StreamTableSinkFactory.class, propertiesMap)
            .createStreamTableSink(propertiesMap);

    String tablePath = tableEnv.getCurrentDatabase() + "." + "PravegaSink";

    ConnectorCatalogTable<?, ?> connectorCatalogTable = ConnectorCatalogTable.sink(sink, false);

    tableEnv.getCatalog(tableEnv.getCurrentCatalog()).get().createTable(
            ObjectPath.fromString(tablePath),
            connectorCatalogTable, false);

    table.insertInto(tablePath);
    env.execute();
}
 

/**
 * Validates the use of Pravega Table Descriptor to generate the source/sink Table factory to
 * write and read from Pravega stream using {@link StreamTableEnvironment}
 * @throws Exception
 */
@Test
public void testStreamingTableUsingDescriptor() throws Exception {

    final String scope = setupUtils.getScope();
    final String streamName = "stream";
    Stream stream = Stream.of(scope, streamName);
    this.setupUtils.createTestStream(stream.getStreamName(), 1);

    StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment().setParallelism(1);
    StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env,
            EnvironmentSettings.newInstance()
                    // watermark is only supported in blink planner
                    .useBlinkPlanner()
                    .inStreamingMode()
                    .build());

    PravegaConfig pravegaConfig = setupUtils.getPravegaConfig();

    Pravega pravega = new Pravega();
    pravega.tableSinkWriterBuilder()
            .withRoutingKeyField("category")
            .forStream(stream)
            .withPravegaConfig(pravegaConfig);
    pravega.tableSourceReaderBuilder()
            .withReaderGroupScope(stream.getScope())
            .forStream(stream)
            .withPravegaConfig(pravegaConfig);

    TableSchema tableSchema = TableSchema.builder()
            .field("category", DataTypes.STRING())
            .field("value", DataTypes.INT())
            .build();

    Schema schema = new Schema().schema(tableSchema);

    ConnectTableDescriptor desc = tableEnv.connect(pravega)
            .withFormat(
                new Json()
                        .failOnMissingField(false)
            )
            .withSchema(schema)
            .inAppendMode();

    desc.createTemporaryTable("test");

    final Map<String, String> propertiesMap = desc.toProperties();
    final TableSink<?> sink = TableFactoryService.find(StreamTableSinkFactory.class, propertiesMap)
            .createStreamTableSink(propertiesMap);
    final TableSource<?> source = TableFactoryService.find(StreamTableSourceFactory.class, propertiesMap)
            .createStreamTableSource(propertiesMap);

    Table table = tableEnv.fromDataStream(env.fromCollection(SAMPLES));

    String tablePathSink = tableEnv.getCurrentDatabase() + "." + "PravegaSink";

    ConnectorCatalogTable<?, ?> connectorCatalogSinkTable = ConnectorCatalogTable.sink(sink, false);

    tableEnv.getCatalog(tableEnv.getCurrentCatalog())
            .get()
            .createTable(
            ObjectPath.fromString(tablePathSink),
            connectorCatalogSinkTable, false);

    table.insertInto("PravegaSink");

    ConnectorCatalogTable<?, ?> connectorCatalogSourceTable = ConnectorCatalogTable.source(source, false);
    String tablePathSource = tableEnv.getCurrentDatabase() + "." + "samples";

    tableEnv.getCatalog(tableEnv.getCurrentCatalog()).get().createTable(
            ObjectPath.fromString(tablePathSource),
            connectorCatalogSourceTable, false);
    // select some sample data from the Pravega-backed table, as a view
    Table view = tableEnv.sqlQuery("SELECT * FROM samples WHERE category IN ('A','B')");

    // write the view to a test sink that verifies the data for test purposes
    tableEnv.toAppendStream(view, SampleRecord.class).addSink(new TestSink(SAMPLES));

    // execute the topology
    try {
        env.execute();
        Assert.fail("expected an exception");
    } catch (Exception e) {
        // we expect the job to fail because the test sink throws a deliberate exception.
        Assert.assertTrue(ExceptionUtils.getRootCause(e) instanceof TestCompletionException);
    }
}
 

@Test
public void testStreamTableSinkUsingDescriptorForAvro() throws Exception {

    // create a Pravega stream for test purposes
    Stream stream = Stream.of(setupUtils.getScope(), "testStreamTableSinkUsingDescriptorForAvro");
    this.setupUtils.createTestStream(stream.getStreamName(), 1);

    // create a Flink Table environment
    StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment().setParallelism(1);
    StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env,
            EnvironmentSettings.newInstance()
                    // watermark is only supported in blink planner
                    .useBlinkPlanner()
                    .inStreamingMode()
                    .build());

    Table table = tableEnv.fromDataStream(env.fromCollection(SAMPLES));

    Pravega pravega = new Pravega();
    pravega.tableSinkWriterBuilder()
            .withRoutingKeyField("category")
            .forStream(stream)
            .withPravegaConfig(setupUtils.getPravegaConfig());

    Avro avro = new Avro();
    String avroSchema =  "{" +
            "  \"type\": \"record\"," +
            "  \"name\": \"test\"," +
            "  \"fields\" : [" +
            "    {\"name\": \"category\", \"type\": \"string\"}," +
            "    {\"name\": \"value\", \"type\": \"int\"}" +
            "  ]" +
            "}";
    avro.avroSchema(avroSchema);

    ConnectTableDescriptor desc = tableEnv.connect(pravega)
            .withFormat(avro)
            .withSchema(new Schema().field("category", DataTypes.STRING()).
                    field("value", DataTypes.INT()))
            .inAppendMode();
    desc.createTemporaryTable("test");

    final Map<String, String> propertiesMap = desc.toProperties();
    final TableSink<?> sink = TableFactoryService.find(StreamTableSinkFactory.class, propertiesMap)
            .createStreamTableSink(propertiesMap);

    String tablePath = tableEnv.getCurrentDatabase() + "." + "PravegaSink";

    ConnectorCatalogTable<?, ?> connectorCatalogTable = ConnectorCatalogTable.sink(sink, false);

    tableEnv.getCatalog(tableEnv.getCurrentCatalog()).get().createTable(
            ObjectPath.fromString(tablePath),
            connectorCatalogTable, false);

    table.insertInto("PravegaSink");
    env.execute();
}
 

private void testTableSourceBatchDescriptor(Stream stream, PravegaConfig pravegaConfig) throws Exception {
    ExecutionEnvironment execEnvRead = ExecutionEnvironment.getExecutionEnvironment();
    // Can only use Legacy Flink planner for BatchTableEnvironment
    BatchTableEnvironment tableEnv = BatchTableEnvironment.create(execEnvRead);
    execEnvRead.setParallelism(1);

    Schema schema = new Schema()
            .field("user", DataTypes.STRING())
            .field("uri", DataTypes.STRING())
            // Note: LocalDateTime is not supported in legacy Flink planner, bridged to Timestamp with the data source.
            // See https://issues.apache.org/jira/browse/FLINK-16693 for more information.
            .field("accessTime", DataTypes.TIMESTAMP(3).bridgedTo(Timestamp.class));

    Pravega pravega = new Pravega();

    pravega.tableSourceReaderBuilder()
            .withReaderGroupScope(stream.getScope())
            .forStream(stream)
            .withPravegaConfig(pravegaConfig);

    ConnectTableDescriptor desc = tableEnv.connect(pravega)
            .withFormat(new Json().failOnMissingField(false))
            .withSchema(schema);

    final Map<String, String> propertiesMap = desc.toProperties();
    final TableSource<?> source = TableFactoryService.find(BatchTableSourceFactory.class, propertiesMap)
            .createBatchTableSource(propertiesMap);

    String tableSourcePath = tableEnv.getCurrentDatabase() + "." + "MyTableRow";

    ConnectorCatalogTable<?, ?> connectorCatalogSourceTable = ConnectorCatalogTable.source(source, true);

    tableEnv.getCatalog(tableEnv.getCurrentCatalog()).get().createTable(
            ObjectPath.fromString(tableSourcePath),
            connectorCatalogSourceTable, false);

    String sqlQuery = "SELECT user, count(uri) from MyTableRow GROUP BY user";

    Table result = tableEnv.sqlQuery(sqlQuery);

    DataSet<Row> resultSet = tableEnv.toDataSet(result, Row.class);

    List<Row> results = resultSet.collect();
    log.info("results: {}", results);

    boolean compare = compare(results, getExpectedResultsRetracted());
    assertTrue("Output does not match expected result", compare);
}
 

private void testTableSourceStreamingDescriptor(Stream stream, PravegaConfig pravegaConfig) throws Exception {
    final StreamExecutionEnvironment execEnvRead = StreamExecutionEnvironment.getExecutionEnvironment();
    execEnvRead.setParallelism(1);
    execEnvRead.enableCheckpointing(100);
    execEnvRead.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);

    StreamTableEnvironment tableEnv = StreamTableEnvironment.create(execEnvRead,
            EnvironmentSettings.newInstance()
                    // watermark is only supported in blink planner
                    .useBlinkPlanner()
                    .inStreamingMode()
                    .build());
    RESULTS.clear();

    // read data from the stream using Table reader
    Schema schema = new Schema()
            .field("user", DataTypes.STRING())
            .field("uri", DataTypes.STRING())
            .field("accessTime", DataTypes.TIMESTAMP(3)).rowtime(
                    new Rowtime().timestampsFromField("accessTime").watermarksPeriodicBounded(30000L));

    Pravega pravega = new Pravega();
    pravega.tableSourceReaderBuilder()
            .withReaderGroupScope(stream.getScope())
            .forStream(stream)
            .withPravegaConfig(pravegaConfig);

    ConnectTableDescriptor desc = tableEnv.connect(pravega)
            .withFormat(new Json().failOnMissingField(true))
            .withSchema(schema)
            .inAppendMode();

    final Map<String, String> propertiesMap = desc.toProperties();
    final TableSource<?> source = TableFactoryService.find(StreamTableSourceFactory.class, propertiesMap)
            .createStreamTableSource(propertiesMap);

    String tableSourcePath = tableEnv.getCurrentDatabase() + "." + "MyTableRow";

    ConnectorCatalogTable<?, ?> connectorCatalogSourceTable = ConnectorCatalogTable.source(source, false);

    tableEnv.getCatalog(tableEnv.getCurrentCatalog()).get().createTable(
            ObjectPath.fromString(tableSourcePath),
            connectorCatalogSourceTable, false);

    String sqlQuery = "SELECT user, " +
            "TUMBLE_END(accessTime, INTERVAL '5' MINUTE) AS accessTime, " +
            "COUNT(uri) AS cnt " +
            "from MyTableRow GROUP BY " +
            "user, TUMBLE(accessTime, INTERVAL '5' MINUTE)";
    Table result = tableEnv.sqlQuery(sqlQuery);

    DataStream<Tuple2<Boolean, Row>> resultSet = tableEnv.toRetractStream(result, Row.class);
    StringSink2 stringSink = new StringSink2(8);
    resultSet.addSink(stringSink);

    try {
        execEnvRead.execute("ReadRowData");
    } catch (Exception e) {
        if (!(ExceptionUtils.getRootCause(e) instanceof SuccessException)) {
            throw e;
        }
    }

    log.info("results: {}", RESULTS);
    boolean compare = compare(RESULTS, getExpectedResultsAppend());
    assertTrue("Output does not match expected result", compare);
}
 

@Override
public RelDataType getRowType(RelDataTypeFactory typeFactory) {
	final FlinkTypeFactory flinkTypeFactory = (FlinkTypeFactory) typeFactory;
	TableSchema tableSchema = lookupResult.getResolvedSchema();
	final DataType[] fieldDataTypes = tableSchema.getFieldDataTypes();
	CatalogBaseTable catalogTable = lookupResult.getTable();
	if (!isStreamingMode
			&& catalogTable instanceof ConnectorCatalogTable
			&& ((ConnectorCatalogTable<?, ?>) catalogTable).getTableSource().isPresent()) {
		// If the table source is bounded, materialize the time attributes to normal TIMESTAMP type.
		// Now for ConnectorCatalogTable, there is no way to
		// deduce if it is bounded in the table environment, so the data types in TableSchema
		// always patched with TimeAttribute.
		// See ConnectorCatalogTable#calculateSourceSchema
		// for details.

		// Remove the patched time attributes type to let the TableSourceTable handle it.
		// We should remove this logic if the isBatch flag in ConnectorCatalogTable is fixed.
		// TODO: Fix FLINK-14844.
		for (int i = 0; i < fieldDataTypes.length; i++) {
			LogicalType lt = fieldDataTypes[i].getLogicalType();
			if (lt instanceof TimestampType
				&& (((TimestampType) lt).getKind() == TimestampKind.PROCTIME
				|| ((TimestampType) lt).getKind() == TimestampKind.ROWTIME)) {
				int precision = ((TimestampType) lt).getPrecision();
				fieldDataTypes[i] = DataTypes.TIMESTAMP(precision);
			}
		}
	}

	// The following block is a workaround to support tables defined by TableEnvironment.connect() and
	// the actual table sources implement DefinedProctimeAttribute/DefinedRowtimeAttributes.
	// It should be removed after we remove DefinedProctimeAttribute/DefinedRowtimeAttributes.
	Optional<TableSource<?>> sourceOpt = findAndCreateTableSource();
	if (isStreamingMode
		&& tableSchema.getTableColumns().stream().noneMatch(TableColumn::isGenerated)
		&& tableSchema.getWatermarkSpecs().isEmpty()
		&& sourceOpt.isPresent()) {
		TableSource<?> source = sourceOpt.get();
		if (TableSourceValidation.hasProctimeAttribute(source)
				|| TableSourceValidation.hasRowtimeAttribute(source)) {
			// If the table is defined by TableEnvironment.connect(), and use the legacy proctime and rowtime
			// descriptors, the TableSchema should fallback to ConnectorCatalogTable#calculateSourceSchema
			tableSchema = ConnectorCatalogTable.calculateSourceSchema(source, false);
		}
	}

	return TableSourceUtil.getSourceRowType(
		flinkTypeFactory,
		tableSchema,
		scala.Option.empty(),
		isStreamingMode);
}
 

@Override
public <U> RelNode visit(TableSourceQueryOperation<U> tableSourceOperation) {
	TableSource<?> tableSource = tableSourceOperation.getTableSource();
	boolean isBatch;
	if (tableSource instanceof LookupableTableSource) {
		isBatch = tableSourceOperation.isBatch();
	} else if (tableSource instanceof StreamTableSource) {
		isBatch = ((StreamTableSource<?>) tableSource).isBounded();
	} else {
		throw new TableException(String.format("%s is not supported.", tableSource.getClass().getSimpleName()));
	}

	FlinkStatistic statistic;
	ObjectIdentifier tableIdentifier;
	if (tableSourceOperation instanceof RichTableSourceQueryOperation &&
		((RichTableSourceQueryOperation<U>) tableSourceOperation).getIdentifier() != null) {
		tableIdentifier = ((RichTableSourceQueryOperation<U>) tableSourceOperation).getIdentifier();
		statistic = ((RichTableSourceQueryOperation<U>) tableSourceOperation).getStatistic();
	} else {
		statistic = FlinkStatistic.UNKNOWN();
		// TableSourceScan requires a unique name of a Table for computing a digest.
		// We are using the identity hash of the TableSource object.
		String refId = "Unregistered_TableSource_" + System.identityHashCode(tableSource);
		CatalogManager catalogManager = relBuilder.getCluster().getPlanner().getContext()
				.unwrap(FlinkContext.class).getCatalogManager();
		tableIdentifier = catalogManager.qualifyIdentifier(UnresolvedIdentifier.of(refId));
	}

	RelDataType rowType = TableSourceUtil.getSourceRowTypeFromSource(
		relBuilder.getTypeFactory(),
		tableSource,
		!isBatch);
	LegacyTableSourceTable<?> tableSourceTable = new LegacyTableSourceTable<>(
		relBuilder.getRelOptSchema(),
		tableIdentifier,
		rowType,
		statistic,
		tableSource,
		!isBatch,
		ConnectorCatalogTable.source(tableSource, isBatch));
	return LogicalTableScan.create(relBuilder.getCluster(), tableSourceTable);
}
 

/**
 * Translate this {@link CatalogSchemaTable} into Flink source table.
 */
private static FlinkPreparingTableBase toPreparingTable(
		RelOptSchema relOptSchema,
		List<String> names,
		RelDataType rowType,
		CatalogSchemaTable schemaTable) {
	final CatalogBaseTable baseTable = schemaTable.getCatalogTable();
	if (baseTable instanceof QueryOperationCatalogView) {
		return convertQueryOperationView(relOptSchema,
			names,
			rowType,
			(QueryOperationCatalogView) baseTable);
	} else if (baseTable instanceof ConnectorCatalogTable) {
		ConnectorCatalogTable<?, ?> connectorTable = (ConnectorCatalogTable<?, ?>) baseTable;
		if ((connectorTable).getTableSource().isPresent()) {
			return convertSourceTable(relOptSchema,
				rowType,
				schemaTable.getTableIdentifier(),
				connectorTable,
				schemaTable.getStatistic(),
				schemaTable.isStreamingMode());
		} else {
			throw new ValidationException("Cannot convert a connector table " +
				"without source.");
		}
	} else if (baseTable instanceof CatalogView) {
		return convertCatalogView(
			relOptSchema,
			names,
			rowType,
			schemaTable.getStatistic(),
			(CatalogView) baseTable);
	} else if (baseTable instanceof CatalogTable) {
		return convertCatalogTable(relOptSchema,
			names,
			rowType,
			(CatalogTable) baseTable,
			schemaTable);
	} else {
		throw new ValidationException("Unsupported table type: " + baseTable);
	}
}
 

@Override
public Catalog createCatalog(String name, Map<String, String> properties) {
	String database = properties.getOrDefault(
		CatalogDescriptorValidator.CATALOG_DEFAULT_DATABASE,
		"default_database");
	GenericInMemoryCatalog genericInMemoryCatalog = new GenericInMemoryCatalog(name, database);

	String tableName = properties.getOrDefault(TEST_TABLE_NAME, TEST_TABLE_NAME);
	StreamTableSource<Row> tableSource = new StreamTableSource<Row>() {
		@Override
		public DataStream<Row> getDataStream(StreamExecutionEnvironment execEnv) {
			return execEnv.fromCollection(TABLE_CONTENTS)
				.returns(new RowTypeInfo(
					new TypeInformation[]{Types.INT(), Types.STRING()},
					new String[]{"id", "string"}));
		}

		@Override
		public TableSchema getTableSchema() {
			return TableSchema.builder()
				.field("id", DataTypes.INT())
				.field("string", DataTypes.STRING())
				.build();
		}

		@Override
		public DataType getProducedDataType() {
			return DataTypes.ROW(
				DataTypes.FIELD("id", DataTypes.INT()),
				DataTypes.FIELD("string", DataTypes.STRING())
			);
		}
	};

	try {
		genericInMemoryCatalog.createTable(
			new ObjectPath(database, tableName),
			ConnectorCatalogTable.source(tableSource, false),
			false
		);
	} catch (Exception e) {
		throw new WrappingRuntimeException(e);
	}

	return genericInMemoryCatalog;
}
 

@Override
public Catalog createCatalog(String name, Map<String, String> properties) {
	String database = properties.getOrDefault(
		CatalogDescriptorValidator.CATALOG_DEFAULT_DATABASE,
		"default_database");
	GenericInMemoryCatalog genericInMemoryCatalog = new GenericInMemoryCatalog(name, database);

	String tableName = properties.getOrDefault(TEST_TABLE_NAME, TEST_TABLE_NAME);
	StreamTableSource<Row> tableSource = new StreamTableSource<Row>() {
		@Override
		public DataStream<Row> getDataStream(StreamExecutionEnvironment execEnv) {
			return execEnv.fromCollection(TABLE_CONTENTS)
				.returns(new RowTypeInfo(
					new TypeInformation[]{Types.INT(), Types.STRING()},
					new String[]{"id", "string"}));
		}

		@Override
		public TableSchema getTableSchema() {
			return TableSchema.builder()
				.field("id", DataTypes.INT())
				.field("string", DataTypes.STRING())
				.build();
		}

		@Override
		public DataType getProducedDataType() {
			return DataTypes.ROW(
				DataTypes.FIELD("id", DataTypes.INT()),
				DataTypes.FIELD("string", DataTypes.STRING())
			);
		}
	};

	try {
		genericInMemoryCatalog.createTable(
			new ObjectPath(database, tableName),
			ConnectorCatalogTable.source(tableSource, false),
			false
		);
	} catch (Exception e) {
		throw new WrappingRuntimeException(e);
	}

	return genericInMemoryCatalog;
}
 

/**
 * Prepare TableEnvironment for query.
 *
 * @param sourceTablePath
 * @return
 */
private static TableEnvironment prepareTableEnv(String sourceTablePath, Boolean useTableStats) {
	//init Table Env
	EnvironmentSettings environmentSettings = EnvironmentSettings
			.newInstance()
			.useBlinkPlanner()
			.inBatchMode()
			.build();
	TableEnvironment tEnv = TableEnvironment.create(environmentSettings);

	//config Optimizer parameters
	tEnv.getConfig().getConfiguration()
			.setInteger(ExecutionConfigOptions.TABLE_EXEC_RESOURCE_DEFAULT_PARALLELISM, 4);
	tEnv.getConfig().getConfiguration()
			.setLong(OptimizerConfigOptions.TABLE_OPTIMIZER_BROADCAST_JOIN_THRESHOLD, 10 * 1024 * 1024);
	tEnv.getConfig().getConfiguration()
			.setBoolean(OptimizerConfigOptions.TABLE_OPTIMIZER_JOIN_REORDER_ENABLED, true);

	//register TPC-DS tables
	TPCDS_TABLES.forEach(table -> {
		TpcdsSchema schema = TpcdsSchemaProvider.getTableSchema(table);
		CsvTableSource.Builder builder = CsvTableSource.builder();
		builder.path(sourceTablePath + FILE_SEPARATOR + table + DATA_SUFFIX);
		for (int i = 0; i < schema.getFieldNames().size(); i++) {
			builder.field(
					schema.getFieldNames().get(i),
					TypeConversions.fromDataTypeToLegacyInfo(schema.getFieldTypes().get(i)));
		}
		builder.fieldDelimiter(COL_DELIMITER);
		builder.emptyColumnAsNull();
		builder.lineDelimiter("\n");
		CsvTableSource tableSource = builder.build();
		ConnectorCatalogTable catalogTable = ConnectorCatalogTable.source(tableSource, true);
		tEnv.getCatalog(tEnv.getCurrentCatalog()).ifPresent(catalog -> {
			try {
				catalog.createTable(new ObjectPath(tEnv.getCurrentDatabase(), table), catalogTable, false);
			} catch (Exception e) {
				throw new RuntimeException(e);
			}
		});
	});
	// register statistics info
	if (useTableStats) {
		TpcdsStatsProvider.registerTpcdsStats(tEnv);
	}
	return tEnv;
}