Java Code Examples for org.apache.flink.table.descriptors.DescriptorProperties#getString()

The following examples show how to use org.apache.flink.table.descriptors.DescriptorProperties#getString() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: KafkaTableSourceSinkFactoryBase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Override
public StreamTableSink<Row> createStreamTableSink(Map<String, String> properties) {
	final DescriptorProperties descriptorProperties = getValidatedProperties(properties);

	final TableSchema schema = descriptorProperties.getTableSchema(SCHEMA());
	final String topic = descriptorProperties.getString(CONNECTOR_TOPIC);
	final Optional<String> proctime = SchemaValidator.deriveProctimeAttribute(descriptorProperties);
	final List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors =
		SchemaValidator.deriveRowtimeAttributes(descriptorProperties);

	// see also FLINK-9870
	if (proctime.isPresent() || !rowtimeAttributeDescriptors.isEmpty() ||
			checkForCustomFieldMapping(descriptorProperties, schema)) {
		throw new TableException("Time attributes and custom field mappings are not supported yet.");
	}

	return createKafkaTableSink(
		schema,
		topic,
		getKafkaProperties(descriptorProperties),
		getFlinkKafkaPartitioner(descriptorProperties),
		getSerializationSchema(properties));
}
Example 2
Source File: KafkaTableSourceSinkFactoryBase.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
public StreamTableSink<Row> createStreamTableSink(Map<String, String> properties) {
	final DescriptorProperties descriptorProperties = getValidatedProperties(properties);

	final TableSchema schema = descriptorProperties.getTableSchema(SCHEMA);
	final String topic = descriptorProperties.getString(CONNECTOR_TOPIC);
	final Optional<String> proctime = SchemaValidator.deriveProctimeAttribute(descriptorProperties);
	final List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors =
		SchemaValidator.deriveRowtimeAttributes(descriptorProperties);

	// see also FLINK-9870
	if (proctime.isPresent() || !rowtimeAttributeDescriptors.isEmpty() ||
			checkForCustomFieldMapping(descriptorProperties, schema)) {
		throw new TableException("Time attributes and custom field mappings are not supported yet.");
	}

	return createKafkaTableSink(
		schema,
		topic,
		getKafkaProperties(descriptorProperties),
		getFlinkKafkaPartitioner(descriptorProperties),
		getSerializationSchema(properties));
}
Example 3
Source File: KafkaTableSourceSinkFactoryBase.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
public StreamTableSource<Row> createStreamTableSource(Map<String, String> properties) {
	final DescriptorProperties descriptorProperties = getValidatedProperties(properties);

	final String topic = descriptorProperties.getString(CONNECTOR_TOPIC);
	final DeserializationSchema<Row> deserializationSchema = getDeserializationSchema(properties);
	final StartupOptions startupOptions = getStartupOptions(descriptorProperties, topic);

	return createKafkaTableSource(
		descriptorProperties.getTableSchema(SCHEMA),
		SchemaValidator.deriveProctimeAttribute(descriptorProperties),
		SchemaValidator.deriveRowtimeAttributes(descriptorProperties),
		SchemaValidator.deriveFieldMapping(
			descriptorProperties,
			Optional.of(deserializationSchema.getProducedType())),
		topic,
		getKafkaProperties(descriptorProperties),
		deserializationSchema,
		startupOptions.startupMode,
		startupOptions.specificOffsets);
}
Example 4
Source File: DatahubTableFactory.java    From alibaba-flink-connectors with Apache License 2.0 6 votes vote down vote up 
@Override
public TableSink<Row> createTableSink(Map<String, String> prop) {
	DescriptorProperties params = new DescriptorProperties();
	params.putProperties(prop);

	new DatahubDescriptorValidator().validate(params);

	TableSchema schema = params.getTableSchema(SCHEMA);

	String project = params.getString(CONNECTOR_PROJECT);
	String topic = params.getString(CONNECTOR_TOPIC);
	String accessId = params.getString(CONNECTOR_ACCESS_ID);
	String accessKey = params.getString(CONNECTOR_ACCESS_KEY);
	String endpoint = params.getString(CONNECTOR_ENDPOINT);

	return new DatahubTableSink(
			project,
			topic,
			accessId,
			accessKey,
			endpoint,
			schema,
			params
	);
}
Example 5
Source File: PulsarCatalogFactory.java    From pulsar-flink with Apache License 2.0 5 votes vote down vote up 
@Override
public Catalog createCatalog(String name, Map<String, String> properties) {
    DescriptorProperties dp = getValidateProperties(properties);
    String defaultDB = dp.getOptionalString(CATALOG_DEFAULT_DATABASE).orElse("public/default");
    String adminUrl = dp.getString(CATALOG_ADMIN_URL);

    return new PulsarCatalog(adminUrl, name, dp.asMap(), defaultDB);
}
Example 6
Source File: TableEntry.java    From flink with Apache License 2.0 5 votes vote down vote up 
private static TableEntry create(DescriptorProperties properties) {
	properties.validateString(TABLES_NAME, false, 1);
	properties.validateEnumValues(
		TABLES_TYPE,
		false,
		Arrays.asList(
			TABLES_TYPE_VALUE_SOURCE,
			TABLES_TYPE_VALUE_SOURCE_TABLE,
			TABLES_TYPE_VALUE_SINK,
			TABLES_TYPE_VALUE_SINK_TABLE,
			TABLES_TYPE_VALUE_BOTH,
			TABLES_TYPE_VALUE_SOURCE_SINK_TABLE,
			TABLES_TYPE_VALUE_VIEW,
			TABLES_TYPE_VALUE_TEMPORAL_TABLE));

	final String name = properties.getString(TABLES_NAME);

	final DescriptorProperties cleanedProperties =
		properties.withoutKeys(Arrays.asList(TABLES_NAME, TABLES_TYPE));

	switch (properties.getString(TABLES_TYPE)) {
		case TABLES_TYPE_VALUE_SOURCE:
		case TABLES_TYPE_VALUE_SOURCE_TABLE:
			return new SourceTableEntry(name, cleanedProperties);
		case TABLES_TYPE_VALUE_SINK:
		case TABLES_TYPE_VALUE_SINK_TABLE:
			return new SinkTableEntry(name, cleanedProperties);
		case TABLES_TYPE_VALUE_BOTH:
		case TABLES_TYPE_VALUE_SOURCE_SINK_TABLE:
			return new SourceSinkTableEntry(name, cleanedProperties);
		case TABLES_TYPE_VALUE_VIEW:
			return new ViewEntry(name, cleanedProperties);
		case TABLES_TYPE_VALUE_TEMPORAL_TABLE:
			return new TemporalTableEntry(name, cleanedProperties);
		default:
			throw new SqlClientException("Unexpected table type.");
	}
}
Example 7
Source File: JdbcTableSourceSinkFactory.java    From flink with Apache License 2.0 5 votes vote down vote up 
private JdbcOptions getJdbcOptions(DescriptorProperties descriptorProperties) {
	final String url = descriptorProperties.getString(CONNECTOR_URL);
	final JdbcOptions.Builder builder = JdbcOptions.builder()
		.setDBUrl(url)
		.setTableName(descriptorProperties.getString(CONNECTOR_TABLE))
		.setDialect(JdbcDialects.get(url).get());

	descriptorProperties.getOptionalString(CONNECTOR_DRIVER).ifPresent(builder::setDriverName);
	descriptorProperties.getOptionalString(CONNECTOR_USERNAME).ifPresent(builder::setUsername);
	descriptorProperties.getOptionalString(CONNECTOR_PASSWORD).ifPresent(builder::setPassword);

	return builder.build();
}
Example 8
Source File: JsonTableSourceFactory.java    From AthenaX with Apache License 2.0 5 votes vote down vote up 
@Override
public StreamTableSource<Row> create(Map<String, String> properties) {
  DescriptorProperties params = new DescriptorProperties(true);
  params.putProperties(properties);
  TableSchema schema = params.getTableSchema(TOPIC_SCHEMA_KEY);
  String topic = params.getString(TOPIC_NAME_KEY);
  Properties conf = new Properties();
  conf.putAll(params.getPrefix(KAFKA_CONFIG_PREFIX));
  return new JsonTableSource(topic, conf, schema);
}
Example 9
Source File: TableEntry.java    From flink with Apache License 2.0 5 votes vote down vote up 
private static TableEntry create(DescriptorProperties properties) {
	properties.validateString(TABLES_NAME, false, 1);
	properties.validateEnumValues(
		TABLES_TYPE,
		false,
		Arrays.asList(
			TABLES_TYPE_VALUE_SOURCE,
			TABLES_TYPE_VALUE_SOURCE_TABLE,
			TABLES_TYPE_VALUE_SINK,
			TABLES_TYPE_VALUE_SINK_TABLE,
			TABLES_TYPE_VALUE_BOTH,
			TABLES_TYPE_VALUE_SOURCE_SINK_TABLE,
			TABLES_TYPE_VALUE_VIEW,
			TABLES_TYPE_VALUE_TEMPORAL_TABLE));

	final String name = properties.getString(TABLES_NAME);

	final DescriptorProperties cleanedProperties =
		properties.withoutKeys(Arrays.asList(TABLES_NAME, TABLES_TYPE));

	switch (properties.getString(TABLES_TYPE)) {
		case TABLES_TYPE_VALUE_SOURCE:
		case TABLES_TYPE_VALUE_SOURCE_TABLE:
			return new SourceTableEntry(name, cleanedProperties);
		case TABLES_TYPE_VALUE_SINK:
		case TABLES_TYPE_VALUE_SINK_TABLE:
			return new SinkTableEntry(name, cleanedProperties);
		case TABLES_TYPE_VALUE_BOTH:
		case TABLES_TYPE_VALUE_SOURCE_SINK_TABLE:
			return new SourceSinkTableEntry(name, cleanedProperties);
		case TABLES_TYPE_VALUE_VIEW:
			return new ViewEntry(name, cleanedProperties);
		case TABLES_TYPE_VALUE_TEMPORAL_TABLE:
			return new TemporalTableEntry(name, cleanedProperties);
		default:
			throw new SqlClientException("Unexpected table type.");
	}
}
Example 10
Source File: TemporalTableEntry.java    From flink with Apache License 2.0 5 votes vote down vote up 
TemporalTableEntry(String name, DescriptorProperties properties) {
	super(name, properties);

	historyTable = properties.getString(TABLES_HISTORY_TABLE);
	primaryKeyFields = properties.getArray(TABLES_PRIMARY_KEY, properties::getString);
	timeAttribute = properties.getString(TABLES_TIME_ATTRIBUTE);
}
Example 11
Source File: AvroRowFormatFactory.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
public SerializationSchema<Row> createSerializationSchema(Map<String, String> properties) {
	final DescriptorProperties descriptorProperties = getValidatedProperties(properties);

	// create and configure
	if (descriptorProperties.containsKey(AvroValidator.FORMAT_RECORD_CLASS)) {
		return new AvroRowSerializationSchema(
			descriptorProperties.getClass(AvroValidator.FORMAT_RECORD_CLASS, SpecificRecord.class));
	} else {
		return new AvroRowSerializationSchema(descriptorProperties.getString(AvroValidator.FORMAT_AVRO_SCHEMA));
	}
}
Example 12
Source File: AvroRowFormatFactory.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
public DeserializationSchema<Row> createDeserializationSchema(Map<String, String> properties) {
	final DescriptorProperties descriptorProperties = getValidatedProperties(properties);

	// create and configure
	if (descriptorProperties.containsKey(AvroValidator.FORMAT_RECORD_CLASS)) {
		return new AvroRowDeserializationSchema(
			descriptorProperties.getClass(AvroValidator.FORMAT_RECORD_CLASS, SpecificRecord.class));
	} else {
		return new AvroRowDeserializationSchema(descriptorProperties.getString(AvroValidator.FORMAT_AVRO_SCHEMA));
	}
}
Example 13
Source File: TemporalTableEntry.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
TemporalTableEntry(String name, DescriptorProperties properties) {
	super(name, properties);

	historyTable = properties.getString(TABLES_HISTORY_TABLE);
	primaryKeyFields = properties.getArray(TABLES_PRIMARY_KEY, properties::getString);
	timeAttribute = properties.getString(TABLES_TIME_ATTRIBUTE);
}
Example 14
Source File: AvroRowFormatFactory.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public SerializationSchema<Row> createSerializationSchema(Map<String, String> properties) {
	final DescriptorProperties descriptorProperties = getValidatedProperties(properties);

	// create and configure
	if (descriptorProperties.containsKey(AvroValidator.FORMAT_RECORD_CLASS)) {
		return new AvroRowSerializationSchema(
			descriptorProperties.getClass(AvroValidator.FORMAT_RECORD_CLASS, SpecificRecord.class));
	} else {
		return new AvroRowSerializationSchema(descriptorProperties.getString(AvroValidator.FORMAT_AVRO_SCHEMA));
	}
}
Example 15
Source File: KuduTableFactory.java    From bahir-flink with Apache License 2.0 5 votes vote down vote up 
@Override
public KuduTableSink createTableSink(Map<String, String> properties) {
    DescriptorProperties descriptorProperties = getValidatedProps(properties);
    String tableName = descriptorProperties.getString(KUDU_TABLE);
    TableSchema schema = descriptorProperties.getTableSchema(SCHEMA);

    return createTableSink(tableName, schema, properties);
}
Example 16
Source File: KuduCatalogFactory.java    From bahir-flink with Apache License 2.0 4 votes vote down vote up 
@Override
public Catalog createCatalog(String name, Map<String, String> properties) {
    final DescriptorProperties descriptorProperties = getValidatedProperties(properties);
    return new KuduCatalog(name, descriptorProperties.getString(KuduTableFactory.KUDU_MASTERS));
}
Example 17
Source File: ViewEntry.java    From flink with Apache License 2.0 4 votes vote down vote up 
ViewEntry(String name, DescriptorProperties properties) {
	super(name, properties);

	query = properties.getString(TABLES_QUERY);
}
Example 18
Source File: CsvTableSinkFactoryBase.java    From flink with Apache License 2.0 4 votes vote down vote up 
protected CsvTableSink createTableSink(
		Boolean isStreaming,
		Map<String, String> properties) {

	DescriptorProperties params = new DescriptorProperties();
	params.putProperties(properties);

	// validate
	new FileSystemValidator().validate(params);
	new OldCsvValidator().validate(params);
	new SchemaValidator(isStreaming, false, false).validate(params);

	// build
	TableSchema tableSchema = TableSchemaUtils.getPhysicalSchema(params.getTableSchema(SCHEMA));

	// if a schema is defined, no matter derive schema is set or not, will use the defined schema
	final boolean hasSchema = params.hasPrefix(FORMAT_FIELDS);
	if (hasSchema) {
		TableSchema formatSchema = params.getTableSchema(FORMAT_FIELDS);
		if (!getFieldLogicalTypes(formatSchema).equals(getFieldLogicalTypes(tableSchema))) {
			throw new TableException(String.format(
					"Encodings that differ from the schema are not supported yet for" +
							" CsvTableSink, format schema is '%s', but table schema is '%s'.",
					formatSchema,
					tableSchema));
		}
	}

	String path = params.getString(CONNECTOR_PATH);
	String fieldDelimiter = params.getOptionalString(FORMAT_FIELD_DELIMITER).orElse(",");
	Optional<String> writeModeParm = params.getOptionalString(FORMAT_WRITE_MODE);
	FileSystem.WriteMode writeMode =
			(writeModeParm.isPresent()) ? FileSystem.WriteMode.valueOf(writeModeParm.get()) : null;
	int numFiles = params.getOptionalInt(FORMAT_NUM_FILES).orElse(-1);

	// bridge to java.sql.Timestamp/Time/Date
	DataType[] dataTypes = Arrays.stream(tableSchema.getFieldDataTypes())
		.map(dt -> {
			switch (dt.getLogicalType().getTypeRoot()) {
				case TIMESTAMP_WITHOUT_TIME_ZONE:
					return dt.bridgedTo(Timestamp.class);
				case TIME_WITHOUT_TIME_ZONE:
					return dt.bridgedTo(Time.class);
				case DATE:
					return dt.bridgedTo(Date.class);
				default:
					return dt;
			}
		})
		.toArray(DataType[]::new);

	return new CsvTableSink(
		path,
		fieldDelimiter,
		numFiles,
		writeMode,
		tableSchema.getFieldNames(),
		dataTypes);
}
Example 19
Source File: ConnectorConfigurations.java    From flink-connectors with Apache License 2.0 4 votes vote down vote up 
@SuppressWarnings("unchecked")
private void populateReaderConfig(DescriptorProperties descriptorProperties) {
    uid = descriptorProperties.getOptionalString(CONNECTOR_READER_READER_GROUP_UID);
    rgScope = descriptorProperties.getOptionalString(CONNECTOR_READER_READER_GROUP_SCOPE);
    rgName = descriptorProperties.getOptionalString(CONNECTOR_READER_READER_GROUP_NAME);
    refreshInterval = descriptorProperties.getOptionalLong(CONNECTOR_READER_READER_GROUP_REFRESH_INTERVAL);
    eventReadTimeoutInterval = descriptorProperties.getOptionalLong(CONNECTOR_READER_READER_GROUP_EVENT_READ_TIMEOUT_INTERVAL);
    checkpointInitiateTimeoutInterval = descriptorProperties.getOptionalLong(CONNECTOR_READER_READER_GROUP_CHECKPOINT_INITIATE_TIMEOUT_INTERVAL);

    final Optional<Class<AssignerWithTimeWindows>> assignerClass = descriptorProperties.getOptionalClass(
            CONNECTOR_READER_USER_TIMESTAMP_ASSIGNER, AssignerWithTimeWindows.class);
    if (assignerClass.isPresent()) {
        assignerWithTimeWindows = Optional.of((AssignerWithTimeWindows<Row>) InstantiationUtil.instantiate(assignerClass.get()));
    } else {
        assignerWithTimeWindows = Optional.empty();
    }

    if (!defaultScope.isPresent() && !rgScope.isPresent()) {
        throw new ValidationException("Must supply either " + CONNECTOR_READER_READER_GROUP_SCOPE + " or " + CONNECTOR_CONNECTION_CONFIG_DEFAULT_SCOPE);
    }

    final List<Map<String, String>> streamPropsList = descriptorProperties.getVariableIndexedProperties(
            CONNECTOR_READER_STREAM_INFO,
            Arrays.asList(CONNECTOR_READER_STREAM_INFO_STREAM));

    if (streamPropsList.isEmpty()) {
        throw new ValidationException(CONNECTOR_READER_STREAM_INFO + " cannot be empty");
    }

    int index = 0;
    for (Map<String, String> propsMap : streamPropsList) {
        if (!propsMap.containsKey(CONNECTOR_READER_STREAM_INFO_SCOPE) && !defaultScope.isPresent()) {
            throw new ValidationException("Must supply either " + CONNECTOR_READER_STREAM_INFO + "." + index + "." + CONNECTOR_READER_STREAM_INFO_SCOPE +
                    " or " + CONNECTOR_CONNECTION_CONFIG_DEFAULT_SCOPE);
        }
        String scopeName = (propsMap.containsKey(CONNECTOR_READER_STREAM_INFO_SCOPE)) ?
                descriptorProperties.getString(propsMap.get(CONNECTOR_READER_STREAM_INFO_SCOPE)) : defaultScope.get();

        if (!propsMap.containsKey(CONNECTOR_READER_STREAM_INFO_STREAM)) {
            throw new ValidationException(CONNECTOR_READER_STREAM_INFO + "." + index + "." +  CONNECTOR_READER_STREAM_INFO_STREAM + " cannot be empty");
        }
        String streamName = descriptorProperties.getString(propsMap.get(CONNECTOR_READER_STREAM_INFO_STREAM));

        String startCut = StreamCut.UNBOUNDED.asText();
        if (propsMap.containsKey(CONNECTOR_READER_STREAM_INFO_START_STREAMCUT)) {
            startCut = descriptorProperties.getString(propsMap.get(CONNECTOR_READER_STREAM_INFO_START_STREAMCUT));
        }

        String endCut = StreamCut.UNBOUNDED.asText();
        if (propsMap.containsKey(CONNECTOR_READER_STREAM_INFO_END_STREAMCUT)) {
            endCut = descriptorProperties.getString(propsMap.get(CONNECTOR_READER_STREAM_INFO_END_STREAMCUT));
        }

        Stream stream = Stream.of(scopeName, streamName);
        readerStreams.add(new StreamWithBoundaries(stream, StreamCut.from(startCut), StreamCut.from(endCut)));
        index++;
    }
}
Example 20
Source File: CatalogEntry.java    From flink with Apache License 2.0 3 votes vote down vote up 
private static CatalogEntry create(DescriptorProperties properties) {
	properties.validateString(CATALOG_NAME, false, 1);

	final String name = properties.getString(CATALOG_NAME);

	final DescriptorProperties cleanedProperties =
		properties.withoutKeys(Collections.singletonList(CATALOG_NAME));

	return new CatalogEntry(name, cleanedProperties);
}