org.apache.flink.table.utils.TableSchemaUtils Java Examples

/**
 * Creates a generic Kafka {@link StreamTableSource}.
 *
 * @param schema                      Schema of the produced table.
 * @param proctimeAttribute           Field name of the processing time attribute.
 * @param rowtimeAttributeDescriptors Descriptor for a rowtime attribute
 * @param fieldMapping                Mapping for the fields of the table schema to
 *                                    fields of the physical returned type.
 * @param topic                       Kafka topic to consume.
 * @param properties                  Properties for the Kafka consumer.
 * @param deserializationSchema       Deserialization schema for decoding records from Kafka.
 * @param startupMode                 Startup mode for the contained consumer.
 * @param specificStartupOffsets      Specific startup offsets; only relevant when startup
 *                                    mode is {@link StartupMode#SPECIFIC_OFFSETS}.
 * @param startupTimestampMillis	  Startup timestamp for offsets; only relevant when startup
 *                                    mode is {@link StartupMode#TIMESTAMP}.
 */
protected KafkaTableSourceBase(
		TableSchema schema,
		Optional<String> proctimeAttribute,
		List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors,
		Optional<Map<String, String>> fieldMapping,
		String topic,
		Properties properties,
		DeserializationSchema<Row> deserializationSchema,
		StartupMode startupMode,
		Map<KafkaTopicPartition, Long> specificStartupOffsets,
		long startupTimestampMillis) {
	this.schema = TableSchemaUtils.checkNoGeneratedColumns(schema);
	this.proctimeAttribute = validateProctimeAttribute(proctimeAttribute);
	this.rowtimeAttributeDescriptors = validateRowtimeAttributeDescriptors(rowtimeAttributeDescriptors);
	this.fieldMapping = fieldMapping;
	this.topic = Preconditions.checkNotNull(topic, "Topic must not be null.");
	this.properties = Preconditions.checkNotNull(properties, "Properties must not be null.");
	this.deserializationSchema = Preconditions.checkNotNull(
		deserializationSchema, "Deserialization schema must not be null.");
	this.startupMode = Preconditions.checkNotNull(startupMode, "Startup mode must not be null.");
	this.specificStartupOffsets = Preconditions.checkNotNull(
		specificStartupOffsets, "Specific offsets must not be null.");
	this.startupTimestampMillis = startupTimestampMillis;
}
 

@Override
public DynamicTableSink createDynamicTableSink(Context context) {
	TableSchema tableSchema = context.getCatalogTable().getSchema();
	ElasticsearchValidationUtils.validatePrimaryKey(tableSchema);
	final FactoryUtil.TableFactoryHelper helper = FactoryUtil.createTableFactoryHelper(this, context);

	final EncodingFormat<SerializationSchema<RowData>> format = helper.discoverEncodingFormat(
		SerializationFormatFactory.class,
		FORMAT_OPTION);

	helper.validate();
	Configuration configuration = new Configuration();
	context.getCatalogTable()
		.getOptions()
		.forEach(configuration::setString);
	Elasticsearch6Configuration config = new Elasticsearch6Configuration(configuration, context.getClassLoader());

	validate(config, configuration);

	return new Elasticsearch6DynamicSink(
		format,
		config,
		TableSchemaUtils.getPhysicalSchema(tableSchema));
}
 

@Override
public StreamTableSink<Tuple2<Boolean, Row>> createStreamTableSink(Map<String, String> properties) {
	final DescriptorProperties descriptorProperties = getValidatedProperties(properties);

	return createElasticsearchUpsertTableSink(
		descriptorProperties.isValue(UPDATE_MODE, UPDATE_MODE_VALUE_APPEND),
		TableSchemaUtils.getPhysicalSchema(descriptorProperties.getTableSchema(SCHEMA)),
		getHosts(descriptorProperties),
		descriptorProperties.getString(CONNECTOR_INDEX),
		descriptorProperties.getString(CONNECTOR_DOCUMENT_TYPE),
		descriptorProperties.getOptionalString(CONNECTOR_KEY_DELIMITER).orElse(DEFAULT_KEY_DELIMITER),
		descriptorProperties.getOptionalString(CONNECTOR_KEY_NULL_LITERAL).orElse(DEFAULT_KEY_NULL_LITERAL),
		getSerializationSchema(properties),
		SUPPORTED_CONTENT_TYPE,
		getFailureHandler(descriptorProperties),
		getSinkOptions(descriptorProperties));
}
 

@Override
public StreamTableSource<Row> createStreamTableSource(Map<String, String> properties) {
	final DescriptorProperties descriptorProperties = getValidatedProperties(properties);
	// create default configuration from current runtime env (`hbase-site.xml` in classpath) first,
	Configuration hbaseClientConf = HBaseConfigurationUtil.getHBaseConfiguration();
	String hbaseZk = descriptorProperties.getString(CONNECTOR_ZK_QUORUM);
	hbaseClientConf.set(HConstants.ZOOKEEPER_QUORUM, hbaseZk);
	descriptorProperties
		.getOptionalString(CONNECTOR_ZK_NODE_PARENT)
		.ifPresent(v -> hbaseClientConf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, v));

	String hTableName = descriptorProperties.getString(CONNECTOR_TABLE_NAME);
	TableSchema tableSchema = TableSchemaUtils.getPhysicalSchema(
		descriptorProperties.getTableSchema(SCHEMA));
	HBaseTableSchema hbaseSchema = validateTableSchema(tableSchema);
	return new HBaseTableSource(hbaseClientConf, hTableName, hbaseSchema, null);
}
 

private void validateCommonProperties(DescriptorProperties properties) {
	properties.validateString(CONNECTOR_URL, false, 1);
	properties.validateString(CONNECTOR_TABLE, false, 1);
	properties.validateString(CONNECTOR_DRIVER, true);
	properties.validateString(CONNECTOR_USERNAME, true);
	properties.validateString(CONNECTOR_PASSWORD, true);

	final String url = properties.getString(CONNECTOR_URL);
	final Optional<JdbcDialect> dialect = JdbcDialects.get(url);
	Preconditions.checkState(dialect.isPresent(), "Cannot handle such jdbc url: " + url);

	TableSchema schema = TableSchemaUtils.getPhysicalSchema(properties.getTableSchema(SCHEMA));
	dialect.get().validate(schema);

	Optional<String> password = properties.getOptionalString(CONNECTOR_PASSWORD);
	if (password.isPresent()) {
		Preconditions.checkArgument(
			properties.getOptionalString(CONNECTOR_USERNAME).isPresent(),
			"Database username must be provided when database password is provided");
	}
}
 

@Override
public DynamicTableSink createDynamicTableSink(Context context) {
	final FactoryUtil.TableFactoryHelper helper = FactoryUtil.createTableFactoryHelper(this, context);
	final ReadableConfig config = helper.getOptions();

	helper.validate();
	validateConfigOptions(config);
	JdbcOptions jdbcOptions = getJdbcOptions(config);
	TableSchema physicalSchema = TableSchemaUtils.getPhysicalSchema(context.getCatalogTable().getSchema());

	return new JdbcDynamicTableSink(
		jdbcOptions,
		getJdbcExecutionOptions(config),
		getJdbcDmlOptions(jdbcOptions, physicalSchema),
		physicalSchema);
}
 

@Override
public StreamTableSink<Tuple2<Boolean, Row>> createStreamTableSink(Map<String, String> properties) {
	DescriptorProperties descriptorProperties = getValidatedProperties(properties);
	TableSchema schema = TableSchemaUtils.getPhysicalSchema(
		descriptorProperties.getTableSchema(SCHEMA));

	final JdbcUpsertTableSink.Builder builder = JdbcUpsertTableSink.builder()
		.setOptions(getJdbcOptions(descriptorProperties))
		.setTableSchema(schema);

	descriptorProperties.getOptionalInt(CONNECTOR_WRITE_FLUSH_MAX_ROWS).ifPresent(builder::setFlushMaxSize);
	descriptorProperties.getOptionalDuration(CONNECTOR_WRITE_FLUSH_INTERVAL).ifPresent(
		s -> builder.setFlushIntervalMills(s.toMillis()));
	descriptorProperties.getOptionalInt(CONNECTOR_WRITE_MAX_RETRIES).ifPresent(builder::setMaxRetryTimes);

	return builder.build();
}
 

@Override
public StreamTableSource<Row> createStreamTableSource(Map<String, String> properties) {
	final DescriptorProperties descriptorProperties = getValidatedProperties(properties);

	final String topic = descriptorProperties.getString(CONNECTOR_TOPIC);
	final DeserializationSchema<Row> deserializationSchema = getDeserializationSchema(properties);
	final StartupOptions startupOptions = getStartupOptions(descriptorProperties, topic);

	return createKafkaTableSource(
		TableSchemaUtils.getPhysicalSchema(descriptorProperties.getTableSchema(SCHEMA)),
		SchemaValidator.deriveProctimeAttribute(descriptorProperties),
		SchemaValidator.deriveRowtimeAttributes(descriptorProperties),
		SchemaValidator.deriveFieldMapping(
			descriptorProperties,
			Optional.of(deserializationSchema.getProducedType())),
		topic,
		getKafkaProperties(descriptorProperties),
		deserializationSchema,
		startupOptions.startupMode,
		startupOptions.specificOffsets,
		startupOptions.startupTimestampMillis);
}
 

@Override
public StreamTableSink<Row> createStreamTableSink(Map<String, String> properties) {
	final DescriptorProperties descriptorProperties = getValidatedProperties(properties);

	final TableSchema schema = TableSchemaUtils.getPhysicalSchema(
		descriptorProperties.getTableSchema(SCHEMA));
	final String topic = descriptorProperties.getString(CONNECTOR_TOPIC);
	final Optional<String> proctime = SchemaValidator.deriveProctimeAttribute(descriptorProperties);
	final List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors =
		SchemaValidator.deriveRowtimeAttributes(descriptorProperties);

	// see also FLINK-9870
	if (proctime.isPresent() || !rowtimeAttributeDescriptors.isEmpty() ||
			checkForCustomFieldMapping(descriptorProperties, schema)) {
		throw new TableException("Time attributes and custom field mappings are not supported yet.");
	}

	return createKafkaTableSink(
		schema,
		topic,
		getKafkaProperties(descriptorProperties),
		getFlinkKafkaPartitioner(descriptorProperties),
		getSerializationSchema(properties));
}
 

@Override
public DynamicTableSink createDynamicTableSink(Context context) {
	TableSchema tableSchema = context.getCatalogTable().getSchema();
	ElasticsearchValidationUtils.validatePrimaryKey(tableSchema);

	final FactoryUtil.TableFactoryHelper helper = FactoryUtil.createTableFactoryHelper(this, context);

	final EncodingFormat<SerializationSchema<RowData>> format = helper.discoverEncodingFormat(
		SerializationFormatFactory.class,
		FORMAT_OPTION);

	helper.validate();
	Configuration configuration = new Configuration();
	context.getCatalogTable()
		.getOptions()
		.forEach(configuration::setString);
	Elasticsearch7Configuration config = new Elasticsearch7Configuration(configuration, context.getClassLoader());

	validate(config, configuration);

	return new Elasticsearch7DynamicSink(
		format,
		config,
		TableSchemaUtils.getPhysicalSchema(tableSchema));
}
 

@Override
public DynamicTableSource createDynamicTableSource(Context context) {
	Configuration options = new Configuration();
	context.getCatalogTable().getOptions().forEach(options::setString);

	TableSchema tableSchema = TableSchemaUtils.getPhysicalSchema(context.getCatalogTable().getSchema());

	DataGenerator[] fieldGenerators = new DataGenerator[tableSchema.getFieldCount()];
	for (int i = 0; i < fieldGenerators.length; i++) {
		fieldGenerators[i] = createDataGenerator(
				tableSchema.getFieldName(i).get(),
				tableSchema.getFieldDataType(i).get(),
				options);
	}

	return new DataGenTableSource(fieldGenerators, tableSchema, options.get(ROWS_PER_SECOND));
}
 

@SuppressWarnings("unchecked")
protected FlinkPravegaTableSink createFlinkPravegaTableSink(Map<String, String> properties) {
    final DescriptorProperties descriptorProperties = getValidatedProperties(properties);
    final TableSchema schema = TableSchemaUtils.getPhysicalSchema(descriptorProperties.getTableSchema(SCHEMA));

    SerializationSchema<Row> serializationSchema = getSerializationSchema(properties);

    ConnectorConfigurations connectorConfigurations = new ConnectorConfigurations();
    connectorConfigurations.parseConfigurations(descriptorProperties, ConnectorConfigurations.ConfigurationType.WRITER);

    TableSinkWriterBuilder tableSinkWriterBuilder = new Pravega().tableSinkWriterBuilder();
    if (connectorConfigurations.getTxnLeaseRenewalInterval().isPresent()) {
        tableSinkWriterBuilder.withTxnLeaseRenewalPeriod(Time.milliseconds(connectorConfigurations.getTxnLeaseRenewalInterval().get().longValue()));
    }
    if (connectorConfigurations.getWriterMode().isPresent()) {
        tableSinkWriterBuilder.withWriterMode(connectorConfigurations.getWriterMode().get());
    }
    if (connectorConfigurations.getMetrics().isPresent()) {
        tableSinkWriterBuilder.enableMetrics(connectorConfigurations.getMetrics().get());
    }
    if (connectorConfigurations.getWatermark().isPresent()) {
        tableSinkWriterBuilder.enableWatermark(connectorConfigurations.getWatermark().get());
    }
    tableSinkWriterBuilder.withPravegaConfig(connectorConfigurations.getPravegaConfig());
    tableSinkWriterBuilder.withRoutingKeyField(connectorConfigurations.getRoutingKey());
    tableSinkWriterBuilder.withSerializationSchema(serializationSchema);
    tableSinkWriterBuilder.forStream(connectorConfigurations.getWriterStream());
    tableSinkWriterBuilder.withPravegaConfig(connectorConfigurations.getPravegaConfig());

    return new FlinkPravegaTableSink(tableSinkWriterBuilder::createSinkFunction,
            tableSinkWriterBuilder::createOutputFormat, schema);
}
 

public HiveTableSink(
		boolean userMrWriter, boolean isBounded, JobConf jobConf, ObjectIdentifier identifier, CatalogTable table) {
	this.userMrWriter = userMrWriter;
	this.isBounded = isBounded;
	this.jobConf = jobConf;
	this.identifier = identifier;
	this.catalogTable = table;
	hiveVersion = Preconditions.checkNotNull(jobConf.get(HiveCatalogValidator.CATALOG_HIVE_VERSION),
			"Hive version is not defined");
	hiveShim = HiveShimLoader.loadHiveShim(hiveVersion);
	tableSchema = TableSchemaUtils.getPhysicalSchema(table.getSchema());
}
 

protected KafkaTableSinkBase(
		TableSchema schema,
		String topic,
		Properties properties,
		Optional<FlinkKafkaPartitioner<Row>> partitioner,
		SerializationSchema<Row> serializationSchema) {
	this.schema = TableSchemaUtils.checkNoGeneratedColumns(schema);
	this.topic = Preconditions.checkNotNull(topic, "Topic must not be null.");
	this.properties = Preconditions.checkNotNull(properties, "Properties must not be null.");
	this.partitioner = Preconditions.checkNotNull(partitioner, "Partitioner must not be null.");
	this.serializationSchema = Preconditions.checkNotNull(serializationSchema, "Serialization schema must not be null.");
}
 

public ElasticsearchUpsertTableSinkBase(
		boolean isAppendOnly,
		TableSchema schema,
		List<Host> hosts,
		String index,
		String docType,
		String keyDelimiter,
		String keyNullLiteral,
		SerializationSchema<Row> serializationSchema,
		XContentType contentType,
		ActionRequestFailureHandler failureHandler,
		Map<SinkOption, String> sinkOptions,
		RequestFactory requestFactory) {

	this.isAppendOnly = isAppendOnly;
	this.schema = TableSchemaUtils.checkNoGeneratedColumns(schema);
	this.hosts = Preconditions.checkNotNull(hosts);
	this.index = Preconditions.checkNotNull(index);
	this.keyDelimiter = Preconditions.checkNotNull(keyDelimiter);
	this.keyNullLiteral = Preconditions.checkNotNull(keyNullLiteral);
	this.docType = Preconditions.checkNotNull(docType);
	this.serializationSchema = Preconditions.checkNotNull(serializationSchema);
	this.contentType = Preconditions.checkNotNull(contentType);
	this.failureHandler = Preconditions.checkNotNull(failureHandler);
	this.sinkOptions = Preconditions.checkNotNull(sinkOptions);
	this.requestFactory = Preconditions.checkNotNull(requestFactory);
}
 

@Override
public DynamicTableSource createDynamicTableSource(Context context) {
	FactoryUtil.TableFactoryHelper helper = FactoryUtil.createTableFactoryHelper(this, context);
	helper.validate();
	ChangelogMode changelogMode = parseChangelogMode(helper.getOptions().get(CHANGELOG_MODE));
	String runtimeSource = helper.getOptions().get(RUNTIME_SOURCE);
	boolean isBounded = helper.getOptions().get(BOUNDED);
	String dataId = helper.getOptions().get(DATA_ID);
	String sourceClass = helper.getOptions().get(TABLE_SOURCE_CLASS);
	boolean isAsync = helper.getOptions().get(ASYNC_ENABLED);
	String lookupFunctionClass = helper.getOptions().get(LOOKUP_FUNCTION_CLASS);
	boolean nestedProjectionSupported = helper.getOptions().get(NESTED_PROJECTION_SUPPORTED);

	if (sourceClass.equals("DEFAULT")) {
		Collection<Row> data = registeredData.getOrDefault(dataId, Collections.emptyList());
		TableSchema physicalSchema = TableSchemaUtils.getPhysicalSchema(context.getCatalogTable().getSchema());
		return new TestValuesTableSource(
			physicalSchema,
			changelogMode,
			isBounded,
			runtimeSource,
			data,
			isAsync,
			lookupFunctionClass,
			nestedProjectionSupported,
			null);
	} else {
		try {
			return InstantiationUtil.instantiate(
				sourceClass,
				DynamicTableSource.class,
				Thread.currentThread().getContextClassLoader());
		} catch (FlinkException e) {
			throw new RuntimeException("Can't instantiate class " + sourceClass, e);
		}
	}
}
 

public CollectStreamTableSink(
		InetAddress targetAddress,
		int targetPort,
		TypeSerializer<Tuple2<Boolean, Row>> serializer,
		TableSchema tableSchema) {
	this.targetAddress = targetAddress;
	this.targetPort = targetPort;
	this.serializer = serializer;
	this.tableSchema = TableSchemaUtils.checkNoGeneratedColumns(tableSchema);
}
 

@Override
public StreamTableSink<Tuple2<Boolean, Row>> createStreamTableSink(Map<String, String> properties) {
	final DescriptorProperties descriptorProperties = getValidatedProperties(properties);
	HBaseOptions.Builder hbaseOptionsBuilder = HBaseOptions.builder();
	hbaseOptionsBuilder.setZkQuorum(descriptorProperties.getString(CONNECTOR_ZK_QUORUM));
	hbaseOptionsBuilder.setTableName(descriptorProperties.getString(CONNECTOR_TABLE_NAME));
	descriptorProperties
		.getOptionalString(CONNECTOR_ZK_NODE_PARENT)
		.ifPresent(hbaseOptionsBuilder::setZkNodeParent);

	TableSchema tableSchema = TableSchemaUtils.getPhysicalSchema(
		descriptorProperties.getTableSchema(SCHEMA));
	HBaseTableSchema hbaseSchema = validateTableSchema(tableSchema);

	HBaseWriteOptions.Builder writeBuilder = HBaseWriteOptions.builder();
	descriptorProperties
		.getOptionalInt(CONNECTOR_WRITE_BUFFER_FLUSH_MAX_ROWS)
		.ifPresent(writeBuilder::setBufferFlushMaxRows);
	descriptorProperties
		.getOptionalMemorySize(CONNECTOR_WRITE_BUFFER_FLUSH_MAX_SIZE)
		.ifPresent(v -> writeBuilder.setBufferFlushMaxSizeInBytes(v.getBytes()));
	descriptorProperties
		.getOptionalDuration(CONNECTOR_WRITE_BUFFER_FLUSH_INTERVAL)
		.ifPresent(v -> writeBuilder.setBufferFlushIntervalMillis(v.toMillis()));

	return new HBaseUpsertTableSink(
		hbaseSchema,
		hbaseOptionsBuilder.build(),
		writeBuilder.build()
	);
}
 

@Override
public DynamicTableSource createDynamicTableSource(Context context) {
	final FactoryUtil.TableFactoryHelper helper = FactoryUtil.createTableFactoryHelper(this, context);
	final ReadableConfig config = helper.getOptions();

	helper.validate();
	validateConfigOptions(config);
	TableSchema physicalSchema = TableSchemaUtils.getPhysicalSchema(context.getCatalogTable().getSchema());
	return new JdbcDynamicTableSource(
		getJdbcOptions(helper.getOptions()),
		getJdbcReadOptions(helper.getOptions()),
		getJdbcLookupOptions(helper.getOptions()),
		physicalSchema);
}
 

@Override
public StreamTableSource<Row> createStreamTableSource(Map<String, String> properties) {
	DescriptorProperties descriptorProperties = getValidatedProperties(properties);
	TableSchema schema = TableSchemaUtils.getPhysicalSchema(
		descriptorProperties.getTableSchema(SCHEMA));

	return JdbcTableSource.builder()
		.setOptions(getJdbcOptions(descriptorProperties))
		.setReadOptions(getJdbcReadOptions(descriptorProperties))
		.setLookupOptions(getJdbcLookupOptions(descriptorProperties))
		.setSchema(schema)
		.build();
}
 

private JdbcUpsertTableSink(
		TableSchema schema,
		JdbcOptions options,
		int flushMaxSize,
		long flushIntervalMills,
		int maxRetryTime) {
	this.schema = TableSchemaUtils.checkNoGeneratedColumns(schema);
	this.options = options;
	this.flushMaxSize = flushMaxSize;
	this.flushIntervalMills = flushIntervalMills;
	this.maxRetryTime = maxRetryTime;
}
 

@Override
public void applyProjection(int[][] projectedFields) {
	TableSchema projectSchema = TableSchemaUtils.projectSchema(
		hbaseSchema.convertsToTableSchema(),
		projectedFields);
	this.hbaseSchema = HBaseTableSchema.fromTableSchema(projectSchema);
}
 

public static TableSchema getSchemaWithSqlTimestamp(TableSchema schema) {
    TableSchema.Builder builder = new TableSchema.Builder();
    TableSchemaUtils.getPhysicalSchema(schema).getTableColumns().forEach(
            tableColumn -> {
                if (tableColumn.getType().getLogicalType() instanceof TimestampType) {
                    builder.field(tableColumn.getName(), tableColumn.getType().bridgedTo(Timestamp.class));
                } else {
                    builder.field(tableColumn.getName(), tableColumn.getType());
                }
            });
    return builder.build();
}
 

/**
 * Creates a Pravega {@link TableSource}.
 * @param sourceFunctionFactory a factory for the {@link FlinkPravegaReader} to implement {@link StreamTableSource}
 * @param inputFormatFactory a factory for the {@link FlinkPravegaInputFormat} to implement {@link BatchTableSource}
 * @param schema the table schema
 */
protected FlinkPravegaTableSource(
        Supplier<FlinkPravegaReader<Row>> sourceFunctionFactory,
        Supplier<FlinkPravegaInputFormat<Row>> inputFormatFactory,
        TableSchema schema) {
    this.sourceFunctionFactory = checkNotNull(sourceFunctionFactory, "sourceFunctionFactory");
    this.inputFormatFactory = checkNotNull(inputFormatFactory, "inputFormatFactory");
    this.schema = TableSchemaUtils.checkNoGeneratedColumns(schema);
}
 

public TestTableSink(TableSchema schema, String property) {
	this.schema = TableSchemaUtils.checkNoGeneratedColumns(schema);
	this.property = property;
}
 

public TestTableSource(TableSchema schema, String property, String proctime, List<RowtimeAttributeDescriptor> rowtime) {
	this.schema = TableSchemaUtils.checkNoGeneratedColumns(schema);
	this.property = property;
	this.proctime = proctime;
	this.rowtime = rowtime;
}
 

@SuppressWarnings("unchecked")
protected FlinkPravegaTableSource createFlinkPravegaTableSource(Map<String, String> properties) {
    final DescriptorProperties descriptorProperties = getValidatedProperties(properties);
    final TableSchema schema = TableSchemaUtils.getPhysicalSchema(descriptorProperties.getTableSchema(SCHEMA));
    final DeserializationSchema<Row> deserializationSchema = getDeserializationSchema(properties);

    ConnectorConfigurations connectorConfigurations = new ConnectorConfigurations();
    connectorConfigurations.parseConfigurations(descriptorProperties, ConnectorConfigurations.ConfigurationType.READER);

    // create source from the reader builder by using the supplied properties
    TableSourceReaderBuilder tableSourceReaderBuilder = new Pravega().tableSourceReaderBuilder();
    tableSourceReaderBuilder.withDeserializationSchema(deserializationSchema);

    if (connectorConfigurations.getAssignerWithTimeWindows().isPresent()) {
        tableSourceReaderBuilder.withTimestampAssigner(connectorConfigurations.getAssignerWithTimeWindows().get());
    }

    if (connectorConfigurations.getUid().isPresent()) {
        tableSourceReaderBuilder.uid(connectorConfigurations.getUid().get());
    }
    if (connectorConfigurations.getRgScope().isPresent()) {
        tableSourceReaderBuilder.withReaderGroupScope(connectorConfigurations.getRgScope().get());
    }
    if (connectorConfigurations.getRgName().isPresent()) {
        tableSourceReaderBuilder.withReaderGroupName(connectorConfigurations.getRgName().get());
    }
    if (connectorConfigurations.getRefreshInterval().isPresent()) {
        tableSourceReaderBuilder.withReaderGroupRefreshTime(Time.milliseconds(connectorConfigurations.getRefreshInterval().get()));
    }
    if (connectorConfigurations.getEventReadTimeoutInterval().isPresent()) {
        tableSourceReaderBuilder.withEventReadTimeout(Time.milliseconds(connectorConfigurations.getEventReadTimeoutInterval().get()));
    }
    if (connectorConfigurations.getCheckpointInitiateTimeoutInterval().isPresent()) {
        tableSourceReaderBuilder.withCheckpointInitiateTimeout(Time.milliseconds(connectorConfigurations.getCheckpointInitiateTimeoutInterval().get()));
    }

    tableSourceReaderBuilder.withPravegaConfig(connectorConfigurations.getPravegaConfig());
    if (connectorConfigurations.getMetrics().isPresent()) {
        tableSourceReaderBuilder.enableMetrics(connectorConfigurations.getMetrics().get());
    }
    tableSourceReaderBuilder.withPravegaConfig(connectorConfigurations.getPravegaConfig());
    for (StreamWithBoundaries streamWithBoundaries: connectorConfigurations.getReaderStreams()) {
        if (streamWithBoundaries.getFrom() != StreamCut.UNBOUNDED && streamWithBoundaries.getTo() != StreamCut.UNBOUNDED) {
            tableSourceReaderBuilder.forStream(streamWithBoundaries.getStream(), streamWithBoundaries.getFrom(), streamWithBoundaries.getTo());
        } else if (streamWithBoundaries.getFrom() != StreamCut.UNBOUNDED) {
            tableSourceReaderBuilder.forStream(streamWithBoundaries.getStream(), streamWithBoundaries.getFrom());
        } else {
            tableSourceReaderBuilder.forStream(streamWithBoundaries.getStream());
        }
    }

    FlinkPravegaTableSource flinkPravegaTableSource = new FlinkPravegaTableSource(
                                                                tableSourceReaderBuilder::buildSourceFunction,
                                                                tableSourceReaderBuilder::buildInputFormat, schema);
    flinkPravegaTableSource.setRowtimeAttributeDescriptors(SchemaValidator.deriveRowtimeAttributes(descriptorProperties));
    Optional<String> procTimeAttribute = SchemaValidator.deriveProctimeAttribute(descriptorProperties);
    if (procTimeAttribute.isPresent()) {
        flinkPravegaTableSource.setProctimeAttribute(procTimeAttribute.get());
    }

    return flinkPravegaTableSource;
}
 

public CollectBatchTableSink(String accumulatorName, TypeSerializer<Row> serializer, TableSchema tableSchema) {
	this.accumulatorName = accumulatorName;
	this.serializer = serializer;
	this.tableSchema = TableSchemaUtils.checkNoGeneratedColumns(tableSchema);
}
 

@Override
public void applyProjection(int[][] projectedFields) {
	this.physicalSchema = TableSchemaUtils.projectSchema(physicalSchema, projectedFields);
	this.projectedFields = Arrays.stream(projectedFields).mapToInt(f -> f[0]).toArray();
}
 

protected CsvTableSource createTableSource(
		Boolean isStreaming,
		Map<String, String> properties) {

	DescriptorProperties params = new DescriptorProperties();
	params.putProperties(properties);

	// validate
	new FileSystemValidator().validate(params);
	new OldCsvValidator().validate(params);
	new SchemaValidator(isStreaming, false, false).validate(params);

	// build
	CsvTableSource.Builder csvTableSourceBuilder = new CsvTableSource.Builder();

	TableSchema tableSchema = TableSchemaUtils.getPhysicalSchema(params.getTableSchema(SCHEMA));

	// if a schema is defined, no matter derive schema is set or not, will use the defined schema
	final boolean hasSchema = params.hasPrefix(FORMAT_FIELDS);
	if (hasSchema) {
		TableSchema formatSchema = params.getTableSchema(FORMAT_FIELDS);
		// the CsvTableSource needs some rework first
		// for now the schema must be equal to the encoding
		// Ignore conversion classes in DataType
		if (!getFieldLogicalTypes(formatSchema).equals(getFieldLogicalTypes(tableSchema))) {
			throw new TableException(String.format(
					"Encodings that differ from the schema are not supported yet for" +
							" CsvTableSource, format schema is '%s', but table schema is '%s'.",
					formatSchema,
					tableSchema));
		}
	}

	params.getOptionalString(CONNECTOR_PATH).ifPresent(csvTableSourceBuilder::path);
	params.getOptionalString(FORMAT_FIELD_DELIMITER).ifPresent(csvTableSourceBuilder::fieldDelimiter);
	params.getOptionalString(FORMAT_LINE_DELIMITER).ifPresent(csvTableSourceBuilder::lineDelimiter);

	for (int i = 0; i < tableSchema.getFieldCount(); ++i) {
		csvTableSourceBuilder.field(tableSchema.getFieldNames()[i], tableSchema.getFieldDataTypes()[i]);
	}
	params.getOptionalCharacter(FORMAT_QUOTE_CHARACTER).ifPresent(csvTableSourceBuilder::quoteCharacter);
	params.getOptionalString(FORMAT_COMMENT_PREFIX).ifPresent(csvTableSourceBuilder::commentPrefix);
	params.getOptionalBoolean(FORMAT_IGNORE_FIRST_LINE).ifPresent(flag -> {
		if (flag) {
			csvTableSourceBuilder.ignoreFirstLine();
		}
	});

	params.getOptionalBoolean(FORMAT_IGNORE_PARSE_ERRORS).ifPresent(flag -> {
		if (flag) {
			csvTableSourceBuilder.ignoreParseErrors();
		}
	});

	return csvTableSourceBuilder.build();
}