org.apache.flink.api.common.serialization.DeserializationSchema Java Examples

The following examples show how to use org.apache.flink.api.common.serialization.DeserializationSchema. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: KafkaTableSourceSinkFactoryBase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Override
public StreamTableSource<Row> createStreamTableSource(Map<String, String> properties) {
	final DescriptorProperties descriptorProperties = getValidatedProperties(properties);

	final String topic = descriptorProperties.getString(CONNECTOR_TOPIC);
	final DeserializationSchema<Row> deserializationSchema = getDeserializationSchema(properties);
	final StartupOptions startupOptions = getStartupOptions(descriptorProperties, topic);

	return createKafkaTableSource(
		descriptorProperties.getTableSchema(SCHEMA()),
		SchemaValidator.deriveProctimeAttribute(descriptorProperties),
		SchemaValidator.deriveRowtimeAttributes(descriptorProperties),
		SchemaValidator.deriveFieldMapping(
			descriptorProperties,
			Optional.of(deserializationSchema.getProducedType())),
		topic,
		getKafkaProperties(descriptorProperties),
		deserializationSchema,
		startupOptions.startupMode,
		startupOptions.specificOffsets);
}
 
Example #2
Source File: Kafka011TableSource.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Creates a Kafka 0.11 {@link StreamTableSource}.
 *
 * @param schema                      Schema of the produced table.
 * @param proctimeAttribute           Field name of the processing time attribute.
 * @param rowtimeAttributeDescriptors Descriptor for a rowtime attribute
 * @param fieldMapping                Mapping for the fields of the table schema to
 *                                    fields of the physical returned type.
 * @param topic                       Kafka topic to consume.
 * @param properties                  Properties for the Kafka consumer.
 * @param deserializationSchema       Deserialization schema for decoding records from Kafka.
 * @param startupMode                 Startup mode for the contained consumer.
 * @param specificStartupOffsets      Specific startup offsets; only relevant when startup
 *                                    mode is {@link StartupMode#SPECIFIC_OFFSETS}.
 */
public Kafka011TableSource(
		TableSchema schema,
		Optional<String> proctimeAttribute,
		List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors,
		Optional<Map<String, String>> fieldMapping,
		String topic,
		Properties properties,
		DeserializationSchema<Row> deserializationSchema,
		StartupMode startupMode,
		Map<KafkaTopicPartition, Long> specificStartupOffsets) {

	super(
		schema,
		proctimeAttribute,
		rowtimeAttributeDescriptors,
		fieldMapping,
		topic,
		properties,
		deserializationSchema,
		startupMode,
		specificStartupOffsets);
}
 
Example #3
Source File: Kafka09TableSource.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Creates a Kafka 0.9 {@link StreamTableSource}.
 *
 * @param schema                      Schema of the produced table.
 * @param proctimeAttribute           Field name of the processing time attribute.
 * @param rowtimeAttributeDescriptors Descriptor for a rowtime attribute
 * @param fieldMapping                Mapping for the fields of the table schema to
 *                                    fields of the physical returned type.
 * @param topic                       Kafka topic to consume.
 * @param properties                  Properties for the Kafka consumer.
 * @param deserializationSchema       Deserialization schema for decoding records from Kafka.
 * @param startupMode                 Startup mode for the contained consumer.
 * @param specificStartupOffsets      Specific startup offsets; only relevant when startup
 *                                    mode is {@link StartupMode#SPECIFIC_OFFSETS}.
 */
public Kafka09TableSource(
		TableSchema schema,
		Optional<String> proctimeAttribute,
		List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors,
		Optional<Map<String, String>> fieldMapping,
		String topic,
		Properties properties,
		DeserializationSchema<Row> deserializationSchema,
		StartupMode startupMode,
		Map<KafkaTopicPartition, Long> specificStartupOffsets) {

	super(
		schema,
		proctimeAttribute,
		rowtimeAttributeDescriptors,
		fieldMapping,
		topic,
		properties,
		deserializationSchema,
		startupMode,
		specificStartupOffsets);
}
 
Example #4
Source File: Kafka010TableSourceSinkFactory.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
protected KafkaTableSourceBase createKafkaTableSource(
		TableSchema schema,
		Optional<String> proctimeAttribute,
		List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors,
		Map<String, String> fieldMapping,
		String topic,
		Properties properties,
		DeserializationSchema<Row> deserializationSchema,
		StartupMode startupMode,
		Map<KafkaTopicPartition, Long> specificStartupOffsets) {

	return new Kafka010TableSource(
		schema,
		proctimeAttribute,
		rowtimeAttributeDescriptors,
		Optional.of(fieldMapping),
		topic,
		properties,
		deserializationSchema,
		startupMode,
		specificStartupOffsets);
}
 
Example #5
Source File: KafkaDynamicTableFactoryBase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public DynamicTableSource createDynamicTableSource(Context context) {
	FactoryUtil.TableFactoryHelper helper = FactoryUtil.createTableFactoryHelper(this, context);

	ReadableConfig tableOptions = helper.getOptions();

	String topic = tableOptions.get(TOPIC);
	DecodingFormat<DeserializationSchema<RowData>> decodingFormat = helper.discoverDecodingFormat(
			DeserializationFormatFactory.class,
			FactoryUtil.FORMAT);
	// Validate the option data type.
	helper.validateExcept(KafkaOptions.PROPERTIES_PREFIX);
	// Validate the option values.
	validateTableOptions(tableOptions);

	DataType producedDataType = context.getCatalogTable().getSchema().toPhysicalRowDataType();
	final KafkaOptions.StartupOptions startupOptions = getStartupOptions(tableOptions, topic);
	return createKafkaTableSource(
			producedDataType,
			topic,
			getKafkaProperties(context.getCatalogTable().getOptions()),
			decodingFormat,
			startupOptions.startupMode,
			startupOptions.specificOffsets,
			startupOptions.startupTimestampMillis);
}
 
Example #6
Source File: Kafka011TableSourceSinkFactory.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
protected KafkaTableSourceBase createKafkaTableSource(
		TableSchema schema,
		Optional<String> proctimeAttribute,
		List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors,
		Map<String, String> fieldMapping,
		String topic,
		Properties properties,
		DeserializationSchema<Row> deserializationSchema,
		StartupMode startupMode,
		Map<KafkaTopicPartition, Long> specificStartupOffsets,
		long startupTimestampMillis) {

	return new Kafka011TableSource(
		schema,
		proctimeAttribute,
		rowtimeAttributeDescriptors,
		Optional.of(fieldMapping),
		topic,
		properties,
		deserializationSchema,
		startupMode,
		specificStartupOffsets,
		startupTimestampMillis);
}
 
Example #7
Source File: Kafka08TableSourceSinkFactoryTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Override
protected KafkaTableSourceBase getExpectedKafkaTableSource(
		TableSchema schema,
		Optional<String> proctimeAttribute,
		List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors,
		Map<String, String> fieldMapping,
		String topic,
		Properties properties,
		DeserializationSchema<Row> deserializationSchema,
		StartupMode startupMode,
		Map<KafkaTopicPartition, Long> specificStartupOffsets) {

	return new Kafka08TableSource(
		schema,
		proctimeAttribute,
		rowtimeAttributeDescriptors,
		Optional.of(fieldMapping),
		topic,
		properties,
		deserializationSchema,
		startupMode,
		specificStartupOffsets
	);
}
 
Example #8
Source File: KafkaTableSourceSinkFactory.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
protected KafkaTableSourceBase createKafkaTableSource(
	TableSchema schema,
	Optional<String> proctimeAttribute,
	List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors,
	Map<String, String> fieldMapping,
	String topic,
	Properties properties,
	DeserializationSchema<Row> deserializationSchema,
	StartupMode startupMode,
	Map<KafkaTopicPartition, Long> specificStartupOffsets,
	long startupTimestampMillis) {

	return new KafkaTableSource(
		schema,
		proctimeAttribute,
		rowtimeAttributeDescriptors,
		Optional.of(fieldMapping),
		topic,
		properties,
		deserializationSchema,
		startupMode,
		specificStartupOffsets,
		startupTimestampMillis);
}
 
Example #9
Source File: KafkaTableSourceBase.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Creates a generic Kafka {@link StreamTableSource}.
 *
 * @param schema                Schema of the produced table.
 * @param topic                 Kafka topic to consume.
 * @param properties            Properties for the Kafka consumer.
 * @param deserializationSchema Deserialization schema for decoding records from Kafka.
 */
protected KafkaTableSourceBase(
		TableSchema schema,
		String topic,
		Properties properties,
		DeserializationSchema<Row> deserializationSchema) {
	this(
		schema,
		Optional.empty(),
		Collections.emptyList(),
		Optional.empty(),
		topic, properties,
		deserializationSchema,
		StartupMode.GROUP_OFFSETS,
		Collections.emptyMap(),
		DEFAULT_STARTUP_TIMESTAMP_MILLIS);
}
 
Example #10
Source File: Kafka011DynamicTableFactoryTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
protected KafkaDynamicSourceBase getExpectedScanSource(
		DataType producedDataType,
		String topic,
		Properties properties,
		DecodingFormat<DeserializationSchema<RowData>> decodingFormat,
		StartupMode startupMode,
		Map<KafkaTopicPartition, Long> specificStartupOffsets,
		long startupTimestamp) {
	return new Kafka011DynamicSource(
			producedDataType,
			topic,
			properties,
			decodingFormat,
			startupMode,
			specificStartupOffsets,
			startupTimestamp);
}
 
Example #11
Source File: Kafka011TableSourceSinkFactoryTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Override
protected KafkaTableSourceBase getExpectedKafkaTableSource(
		TableSchema schema,
		Optional<String> proctimeAttribute,
		List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors,
		Map<String, String> fieldMapping,
		String topic,
		Properties properties,
		DeserializationSchema<Row> deserializationSchema,
		StartupMode startupMode,
		Map<KafkaTopicPartition, Long> specificStartupOffsets) {

	return new Kafka011TableSource(
		schema,
		proctimeAttribute,
		rowtimeAttributeDescriptors,
		Optional.of(fieldMapping),
		topic,
		properties,
		deserializationSchema,
		startupMode,
		specificStartupOffsets
	);
}
 
Example #12
Source File: KafkaTableSourceBase.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Creates a generic Kafka {@link StreamTableSource}.
 *
 * @param schema                      Schema of the produced table.
 * @param proctimeAttribute           Field name of the processing time attribute.
 * @param rowtimeAttributeDescriptors Descriptor for a rowtime attribute
 * @param fieldMapping                Mapping for the fields of the table schema to
 *                                    fields of the physical returned type.
 * @param topic                       Kafka topic to consume.
 * @param properties                  Properties for the Kafka consumer.
 * @param deserializationSchema       Deserialization schema for decoding records from Kafka.
 * @param startupMode                 Startup mode for the contained consumer.
 * @param specificStartupOffsets      Specific startup offsets; only relevant when startup
 *                                    mode is {@link StartupMode#SPECIFIC_OFFSETS}.
 * @param startupTimestampMillis	  Startup timestamp for offsets; only relevant when startup
 *                                    mode is {@link StartupMode#TIMESTAMP}.
 */
protected KafkaTableSourceBase(
		TableSchema schema,
		Optional<String> proctimeAttribute,
		List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors,
		Optional<Map<String, String>> fieldMapping,
		String topic,
		Properties properties,
		DeserializationSchema<Row> deserializationSchema,
		StartupMode startupMode,
		Map<KafkaTopicPartition, Long> specificStartupOffsets,
		long startupTimestampMillis) {
	this.schema = TableSchemaUtils.checkNoGeneratedColumns(schema);
	this.proctimeAttribute = validateProctimeAttribute(proctimeAttribute);
	this.rowtimeAttributeDescriptors = validateRowtimeAttributeDescriptors(rowtimeAttributeDescriptors);
	this.fieldMapping = fieldMapping;
	this.topic = Preconditions.checkNotNull(topic, "Topic must not be null.");
	this.properties = Preconditions.checkNotNull(properties, "Properties must not be null.");
	this.deserializationSchema = Preconditions.checkNotNull(
		deserializationSchema, "Deserialization schema must not be null.");
	this.startupMode = Preconditions.checkNotNull(startupMode, "Startup mode must not be null.");
	this.specificStartupOffsets = Preconditions.checkNotNull(
		specificStartupOffsets, "Specific offsets must not be null.");
	this.startupTimestampMillis = startupTimestampMillis;
}
 
Example #13
Source File: JsonRowFormatFactory.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public DeserializationSchema<Row> createDeserializationSchema(Map<String, String> properties) {
	final DescriptorProperties descriptorProperties = getValidatedProperties(properties);

	// create and configure
	final JsonRowDeserializationSchema.Builder schema =
		new JsonRowDeserializationSchema.Builder(createTypeInformation(descriptorProperties));

	descriptorProperties.getOptionalBoolean(JsonValidator.FORMAT_FAIL_ON_MISSING_FIELD)
		.ifPresent(flag -> {
			if (flag) {
				schema.failOnMissingField();
			}
		});
	descriptorProperties.getOptionalBoolean(JsonValidator.FORMAT_IGNORE_PARSE_ERRORS)
		.ifPresent(flag -> {
			if (flag) {
				schema.ignoreParseErrors();
			}
		});
	return schema.build();
}
 
Example #14
Source File: Kafka011TableSource.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Creates a Kafka 0.11 {@link StreamTableSource}.
 *
 * @param schema                      Schema of the produced table.
 * @param proctimeAttribute           Field name of the processing time attribute.
 * @param rowtimeAttributeDescriptors Descriptor for a rowtime attribute
 * @param fieldMapping                Mapping for the fields of the table schema to
 *                                    fields of the physical returned type.
 * @param topic                       Kafka topic to consume.
 * @param properties                  Properties for the Kafka consumer.
 * @param deserializationSchema       Deserialization schema for decoding records from Kafka.
 * @param startupMode                 Startup mode for the contained consumer.
 * @param specificStartupOffsets      Specific startup offsets; only relevant when startup
 *                                    mode is {@link StartupMode#SPECIFIC_OFFSETS}.
 */
public Kafka011TableSource(
		TableSchema schema,
		Optional<String> proctimeAttribute,
		List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors,
		Optional<Map<String, String>> fieldMapping,
		String topic,
		Properties properties,
		DeserializationSchema<Row> deserializationSchema,
		StartupMode startupMode,
		Map<KafkaTopicPartition, Long> specificStartupOffsets) {

	super(
		schema,
		proctimeAttribute,
		rowtimeAttributeDescriptors,
		fieldMapping,
		topic,
		properties,
		deserializationSchema,
		startupMode,
		specificStartupOffsets);
}
 
Example #15
Source File: Kafka010TableSourceSinkFactoryTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Override
protected KafkaTableSourceBase getExpectedKafkaTableSource(
		TableSchema schema,
		Optional<String> proctimeAttribute,
		List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors,
		Map<String, String> fieldMapping,
		String topic,
		Properties properties,
		DeserializationSchema<Row> deserializationSchema,
		StartupMode startupMode,
		Map<KafkaTopicPartition, Long> specificStartupOffsets) {

	return new Kafka010TableSource(
		schema,
		proctimeAttribute,
		rowtimeAttributeDescriptors,
		Optional.of(fieldMapping),
		topic,
		properties,
		deserializationSchema,
		startupMode,
		specificStartupOffsets
	);
}
 
Example #16
Source File: KafkaTableSourceBase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Creates a generic Kafka {@link StreamTableSource}.
 *
 * @param schema                      Schema of the produced table.
 * @param proctimeAttribute           Field name of the processing time attribute.
 * @param rowtimeAttributeDescriptors Descriptor for a rowtime attribute
 * @param fieldMapping                Mapping for the fields of the table schema to
 *                                    fields of the physical returned type.
 * @param topic                       Kafka topic to consume.
 * @param properties                  Properties for the Kafka consumer.
 * @param deserializationSchema       Deserialization schema for decoding records from Kafka.
 * @param startupMode                 Startup mode for the contained consumer.
 * @param specificStartupOffsets      Specific startup offsets; only relevant when startup
 *                                    mode is {@link StartupMode#SPECIFIC_OFFSETS}.
 */
protected KafkaTableSourceBase(
		TableSchema schema,
		Optional<String> proctimeAttribute,
		List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors,
		Optional<Map<String, String>> fieldMapping,
		String topic,
		Properties properties,
		DeserializationSchema<Row> deserializationSchema,
		StartupMode startupMode,
		Map<KafkaTopicPartition, Long> specificStartupOffsets) {
	this.schema = Preconditions.checkNotNull(schema, "Schema must not be null.");
	this.proctimeAttribute = validateProctimeAttribute(proctimeAttribute);
	this.rowtimeAttributeDescriptors = validateRowtimeAttributeDescriptors(rowtimeAttributeDescriptors);
	this.fieldMapping = fieldMapping;
	this.topic = Preconditions.checkNotNull(topic, "Topic must not be null.");
	this.properties = Preconditions.checkNotNull(properties, "Properties must not be null.");
	this.deserializationSchema = Preconditions.checkNotNull(
		deserializationSchema, "Deserialization schema must not be null.");
	this.startupMode = Preconditions.checkNotNull(startupMode, "Startup mode must not be null.");
	this.specificStartupOffsets = Preconditions.checkNotNull(
		specificStartupOffsets, "Specific offsets must not be null.");
}
 
Example #17
Source File: CsvRowFormatFactoryTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testSchemaDerivation() {
	final Map<String, String> properties = new HashMap<>();
	properties.putAll(new Schema().schema(TableSchema.fromTypeInfo(SCHEMA)).toProperties());
	properties.putAll(new Csv().deriveSchema().toProperties());

	final CsvRowSerializationSchema expectedSer = new CsvRowSerializationSchema.Builder(SCHEMA).build();
	final CsvRowDeserializationSchema expectedDeser = new CsvRowDeserializationSchema.Builder(SCHEMA).build();

	final SerializationSchema<?> actualSer = TableFactoryService
		.find(SerializationSchemaFactory.class, properties)
		.createSerializationSchema(properties);

	assertEquals(expectedSer, actualSer);

	final DeserializationSchema<?> actualDeser = TableFactoryService
		.find(DeserializationSchemaFactory.class, properties)
		.createDeserializationSchema(properties);

	assertEquals(expectedDeser, actualDeser);
}
 
Example #18
Source File: Kafka010DynamicTableFactory.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
protected KafkaDynamicSourceBase createKafkaTableSource(
		DataType producedDataType,
		String topic,
		Properties properties,
		DecodingFormat<DeserializationSchema<RowData>> decodingFormat,
		StartupMode startupMode,
		Map<KafkaTopicPartition, Long> specificStartupOffsets,
		long startupTimestampMillis) {

	return new Kafka010DynamicSource(
		producedDataType,
		topic,
		properties,
		decodingFormat,
		startupMode,
		specificStartupOffsets,
		startupTimestampMillis);
}
 
Example #19
Source File: Kafka011TableSourceSinkFactory.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
protected KafkaTableSourceBase createKafkaTableSource(
		TableSchema schema,
		Optional<String> proctimeAttribute,
		List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors,
		Map<String, String> fieldMapping,
		String topic,
		Properties properties,
		DeserializationSchema<Row> deserializationSchema,
		StartupMode startupMode,
		Map<KafkaTopicPartition, Long> specificStartupOffsets) {

	return new Kafka011TableSource(
		schema,
		proctimeAttribute,
		rowtimeAttributeDescriptors,
		Optional.of(fieldMapping),
		topic,
		properties,
		deserializationSchema,
		startupMode,
		specificStartupOffsets);
}
 
Example #20
Source File: Kafka08TableSource.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a Kafka 0.8 {@link StreamTableSource}.
 *
 * @param schema                Schema of the produced table.
 * @param topic                 Kafka topic to consume.
 * @param properties            Properties for the Kafka consumer.
 * @param deserializationSchema Deserialization schema for decoding records from Kafka.
 */
public Kafka08TableSource(
		TableSchema schema,
		String topic,
		Properties properties,
		DeserializationSchema<Row> deserializationSchema) {

	super(schema, topic, properties, deserializationSchema);
}
 
Example #21
Source File: Kafka09TableSource.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a Kafka 0.9 {@link StreamTableSource}.
 *
 * @param schema                Schema of the produced table.
 * @param topic                 Kafka topic to consume.
 * @param properties            Properties for the Kafka consumer.
 * @param deserializationSchema Deserialization schema for decoding records from Kafka.
 */
public Kafka09TableSource(
		TableSchema schema,
		String topic,
		Properties properties,
		DeserializationSchema<Row> deserializationSchema) {

	super(schema, topic, properties, deserializationSchema);
}
 
Example #22
Source File: JsonRowFormatFactoryTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private void testJsonSchemaDeserializationSchema(Map<String, String> properties) {
	final DeserializationSchema<?> actual2 = TableFactoryService
		.find(DeserializationSchemaFactory.class, properties)
		.createDeserializationSchema(properties);
	final JsonRowDeserializationSchema expected2 = new JsonRowDeserializationSchema.Builder(JSON_SCHEMA)
		.failOnMissingField()
		.build();
	assertEquals(expected2, actual2);
}
 
Example #23
Source File: Kafka010TableSource.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a Kafka 0.10 {@link StreamTableSource}.
 *
 * @param schema                Schema of the produced table.
 * @param topic                 Kafka topic to consume.
 * @param properties            Properties for the Kafka consumer.
 * @param deserializationSchema Deserialization schema for decoding records from Kafka.
 */
public Kafka010TableSource(
		TableSchema schema,
		String topic,
		Properties properties,
		DeserializationSchema<Row> deserializationSchema) {

	super(schema, topic, properties, deserializationSchema);
}
 
Example #24
Source File: KafkaTableSourceBase.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * NOTE: This method is for internal use only for defining a TableSource.
 *       Do not use it in Table API programs.
 */
@Override
public DataStream<Row> getDataStream(StreamExecutionEnvironment env) {

	DeserializationSchema<Row> deserializationSchema = getDeserializationSchema();
	// Version-specific Kafka consumer
	FlinkKafkaConsumerBase<Row> kafkaConsumer = getKafkaConsumer(topic, properties, deserializationSchema);
	return env.addSource(kafkaConsumer).name(explainSource());
}
 
Example #25
Source File: KafkaTableSourceBase.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Returns a version-specific Kafka consumer with the start position configured.
 *
 * @param topic                 Kafka topic to consume.
 * @param properties            Properties for the Kafka consumer.
 * @param deserializationSchema Deserialization schema to use for Kafka records.
 * @return The version-specific Kafka consumer
 */
protected FlinkKafkaConsumerBase<Row> getKafkaConsumer(
		String topic,
		Properties properties,
		DeserializationSchema<Row> deserializationSchema) {
	FlinkKafkaConsumerBase<Row> kafkaConsumer =
			createKafkaConsumer(topic, properties, deserializationSchema);
	switch (startupMode) {
		case EARLIEST:
			kafkaConsumer.setStartFromEarliest();
			break;
		case LATEST:
			kafkaConsumer.setStartFromLatest();
			break;
		case GROUP_OFFSETS:
			kafkaConsumer.setStartFromGroupOffsets();
			break;
		case SPECIFIC_OFFSETS:
			kafkaConsumer.setStartFromSpecificOffsets(specificStartupOffsets);
			break;
		case TIMESTAMP:
			kafkaConsumer.setStartFromTimestamp(startupTimestampMillis);
			break;
	}
	kafkaConsumer.setCommitOffsetsOnCheckpoints(properties.getProperty("group.id") != null);
	return kafkaConsumer;
}
 
Example #26
Source File: KafkaTableSource.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
protected FlinkKafkaConsumerBase<Row> createKafkaConsumer(
	String topic,
	Properties properties,
	DeserializationSchema<Row> deserializationSchema) {

	return new FlinkKafkaConsumer<Row>(topic, deserializationSchema, properties);
}
 
Example #27
Source File: TestFormatFactory.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public DecodingFormat<DeserializationSchema<RowData>> createDecodingFormat(
		DynamicTableFactory.Context context,
		ReadableConfig formatConfig) {
	FactoryUtil.validateFactoryOptions(this, formatConfig);
	return new DecodingFormatMock(formatConfig.get(DELIMITER), formatConfig.get(FAIL_ON_MISSING));
}
 
Example #28
Source File: JsonRowFormatFactory.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public DeserializationSchema<Row> createDeserializationSchema(Map<String, String> properties) {
	final DescriptorProperties descriptorProperties = getValidatedProperties(properties);

	// create and configure
	final JsonRowDeserializationSchema schema = new JsonRowDeserializationSchema(createTypeInformation(descriptorProperties));

	descriptorProperties.getOptionalBoolean(JsonValidator.FORMAT_FAIL_ON_MISSING_FIELD)
			.ifPresent(schema::setFailOnMissingField);

	return schema;
}
 
Example #29
Source File: AvroDeserializationSchemaTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testSpecificRecordWithConfluentSchemaRegistry() throws Exception {
	DeserializationSchema<Address> deserializer = AvroDeserializationSchema.forSpecific(Address.class);

	byte[] encodedAddress = writeRecord(address, Address.getClassSchema());
	Address deserializedAddress = deserializer.deserialize(encodedAddress);
	assertEquals(address, deserializedAddress);
}
 
Example #30
Source File: KafkaTableSourceSinkFactoryBase.java    From flink with Apache License 2.0 5 votes vote down vote up
private DeserializationSchema<Row> getDeserializationSchema(Map<String, String> properties) {
	@SuppressWarnings("unchecked")
	final DeserializationSchemaFactory<Row> formatFactory = TableFactoryService.find(
		DeserializationSchemaFactory.class,
		properties,
		this.getClass().getClassLoader());
	return formatFactory.createDeserializationSchema(properties);
}