org.apache.flink.table.sources.tsextractors.ExistingField Java Examples

@Test
public void buildKafkaSource() throws Exception {
    File file = ResourceUtils.getFile("classpath:yaml/kafka-source.yaml");
    SourceDescriptor sourceDescriptor = BindPropertiesUtil.bindProperties(file, SourceDescriptor.class);
    Kafka010ConnectorDescriptor connectorDescriptor = BindPropertiesUtil.bindProperties(sourceDescriptor.getConnector(), Kafka010ConnectorDescriptor.class);
    assertThat(connectorDescriptor.getTopic()).isEqualTo("app-log");
    assertThat(connectorDescriptor.getStartupMode()).isEqualTo("earliest-offset");
    assertThat(connectorDescriptor.getSpecificOffsets().get("1")).isEqualTo("1000");
    assertThat(connectorDescriptor.getSpecificOffsets().get("2")).isEqualTo("3000");
    assertThat(connectorDescriptor.getProperties().get("bootstrap.servers")).isEqualTo("127.0.0.1:9092");
    assertThat(connectorDescriptor.getProperties().get("group.id")).isEqualTo("testGroup");
    assertThat(sourceDescriptor.getSchema()).isNotNull();
    FormatDescriptor formatDescriptor = sourceDescriptor.getFormat();
    KafkaTableSourceBase alchemyKafkaTableSource = connectorDescriptor.buildSource(sourceDescriptor.getSchema(), formatDescriptor);
    assertThat(alchemyKafkaTableSource).isNotNull();
    assertThat(alchemyKafkaTableSource.getProctimeAttribute()).isEqualTo("procTime");
    List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors = alchemyKafkaTableSource.getRowtimeAttributeDescriptors();
    assertThat(rowtimeAttributeDescriptors).isNotNull();
    assertThat(rowtimeAttributeDescriptors.get(0).getAttributeName()).isEqualTo("rowTime");
    assertThat(rowtimeAttributeDescriptors.get(0).getTimestampExtractor()).isInstanceOf(ExistingField.class);
    assertThat(rowtimeAttributeDescriptors.get(0).getWatermarkStrategy()).isInstanceOf(BoundedOutOfOrderTimestamps.class);
    DeserializationSchema deserializationSchema = formatDescriptor.transform(new Tuple2<>(alchemyKafkaTableSource.getReturnType(), true));
    assertThat(deserializationSchema).isInstanceOf(GrokRowDeserializationSchema.class);
}
 

@Override
public List<RowtimeAttributeDescriptor> getRowtimeAttributeDescriptors() {
	return Collections.singletonList(
		new RowtimeAttributeDescriptor(
			"rowtime",
			new ExistingField("ts"),
			new BoundedOutOfOrderTimestamps(100)));
}
 

@Override
public List<RowtimeAttributeDescriptor> getRowtimeAttributeDescriptors() {
	return Collections.singletonList(
		new RowtimeAttributeDescriptor(
			"timestamp",
			new ExistingField("timestamp"),
			new BoundedOutOfOrderTimestamps(100)));
}
 

@Override
public List<RowtimeAttributeDescriptor> getRowtimeAttributeDescriptors() {
	return Collections.singletonList(
		new RowtimeAttributeDescriptor(
			"rowtime",
			new ExistingField("ts"),
			new BoundedOutOfOrderTimestamps(100)));
}
 

public TimestampExtractor get(){
    if(type == null){
        return null;
    }
    if(type.equals(Type.FIELD.getType())){
        return new ExistingField(from);
    }else if(type.equals(Type.SOURCE.getType())){
        return new StreamRecordTimestamp();
    }
    return null;
}
 

@Override
public void handleRequest() {

    TableSchema tableSchema = TripRecord.getTableSchema();

    FlinkPravegaJsonTableSource source = FlinkPravegaJsonTableSource.builder()
            .forStream(Stream.of(getScope(), getStream()).getScopedName())
            .withPravegaConfig(getPravegaConfig())
            .failOnMissingField(true)
            .withRowtimeAttribute("pickupTime", new ExistingField("pickupTime"), new BoundedOutOfOrderTimestamps(30000L))
            .withSchema(tableSchema)
            .withReaderGroupScope(getScope())
            .build();

    StreamExecutionEnvironment env = getStreamExecutionEnvironment();

    // create a TableEnvironment
    StreamTableEnvironment tEnv = StreamTableEnvironment.create(env);
    tEnv.registerTableSource("TaxiRide", source);

    String fields = "vendorId, pickupTime, startLocationId, destLocationId, startLocationBorough, startLocationZone, destLocationBorough, destLocationZone";

    Table popularRides = tEnv
            .scan("TaxiRide")
            .select(fields)
            .window(Slide.over("15.minutes").every("5.minutes").on("pickupTime").as("w"))
            .groupBy("vendorId, w")
            .select("vendorId, w.start AS start, w.end AS end, count(vendorId) AS cnt");

    tEnv.toAppendStream(popularRides, Row.class).print();

    try {
        env.execute("Popular-Taxi-Vendor");
    } catch (Exception e) {
        log.error("Application Failed", e);
    }
}
 

@Override
public void handleRequest() {

    TableSchema tableSchema = TripRecord.getTableSchema();

    FlinkPravegaJsonTableSource source = FlinkPravegaJsonTableSource.builder()
            .forStream(Stream.of(getScope(), getStream()).getScopedName())
            .withPravegaConfig(getPravegaConfig())
            .failOnMissingField(true)
            .withRowtimeAttribute("dropOffTime", new ExistingField("dropOffTime"), new BoundedOutOfOrderTimestamps(30000L))
            .withSchema(tableSchema)
            .withReaderGroupScope(getScope())
            .build();

    StreamExecutionEnvironment env = getStreamExecutionEnvironment();

    // create a TableEnvironment
    StreamTableEnvironment tEnv = StreamTableEnvironment.create(env);
    tEnv.registerTableSource("TaxiRide", source);

    String fields = "passengerCount, dropOffTime, destLocationZone";

    Table noOfTravelersPerDest = tEnv
            .scan("TaxiRide")
            .select(fields)
            .window(Tumble.over("1.hour").on("dropOffTime").as("w"))
            .groupBy("destLocationZone, w")
            .select("destLocationZone, w.start AS start, w.end AS end, count(passengerCount) AS cnt");

    tEnv.toAppendStream(noOfTravelersPerDest, Row.class).print();

    try {
        env.execute("Max-Travellers-Per-Destination");
    } catch (Exception e) {
        log.error("Application Failed", e);
    }
}
 

@Override
public List<RowtimeAttributeDescriptor> getRowtimeAttributeDescriptors() {
	return Collections.singletonList(
		new RowtimeAttributeDescriptor(
			"rowtime",
			new ExistingField("ts"),
			new BoundedOutOfOrderTimestamps(100)));
}
 

@Test
@SuppressWarnings("unchecked")
public void testTableSource() {

	// prepare parameters for Kafka table source

	final TableSchema schema = TableSchema.builder()
		.field(FRUIT_NAME, Types.STRING())
		.field(COUNT, Types.DECIMAL())
		.field(EVENT_TIME, Types.SQL_TIMESTAMP())
		.field(PROC_TIME, Types.SQL_TIMESTAMP())
		.build();

	final List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors = Collections.singletonList(
		new RowtimeAttributeDescriptor(EVENT_TIME, new ExistingField(TIME), new AscendingTimestamps()));

	final Map<String, String> fieldMapping = new HashMap<>();
	fieldMapping.put(FRUIT_NAME, NAME);
	fieldMapping.put(NAME, NAME);
	fieldMapping.put(COUNT, COUNT);
	fieldMapping.put(TIME, TIME);

	final Map<KafkaTopicPartition, Long> specificOffsets = new HashMap<>();
	specificOffsets.put(new KafkaTopicPartition(TOPIC, PARTITION_0), OFFSET_0);
	specificOffsets.put(new KafkaTopicPartition(TOPIC, PARTITION_1), OFFSET_1);

	final TestDeserializationSchema deserializationSchema = new TestDeserializationSchema(
		TableSchema.builder()
			.field(NAME, Types.STRING())
			.field(COUNT, Types.DECIMAL())
			.field(TIME, Types.SQL_TIMESTAMP())
			.build()
			.toRowType()
	);

	final KafkaTableSourceBase expected = getExpectedKafkaTableSource(
		schema,
		Optional.of(PROC_TIME),
		rowtimeAttributeDescriptors,
		fieldMapping,
		TOPIC,
		KAFKA_PROPERTIES,
		deserializationSchema,
		StartupMode.SPECIFIC_OFFSETS,
		specificOffsets);

	TableSourceUtil.validateTableSource(expected);

	// construct table source using descriptors and table source factory

	final TestTableDescriptor testDesc = new TestTableDescriptor(
			new Kafka()
				.version(getKafkaVersion())
				.topic(TOPIC)
				.properties(KAFKA_PROPERTIES)
				.sinkPartitionerRoundRobin() // test if accepted although not needed
				.startFromSpecificOffsets(OFFSETS))
		.withFormat(new TestTableFormat())
		.withSchema(
			new Schema()
				.field(FRUIT_NAME, Types.STRING()).from(NAME)
				.field(COUNT, Types.DECIMAL()) // no from so it must match with the input
				.field(EVENT_TIME, Types.SQL_TIMESTAMP()).rowtime(
					new Rowtime().timestampsFromField(TIME).watermarksPeriodicAscending())
				.field(PROC_TIME, Types.SQL_TIMESTAMP()).proctime())
		.inAppendMode();

	final Map<String, String> propertiesMap = testDesc.toProperties();
	final TableSource<?> actualSource = TableFactoryService.find(StreamTableSourceFactory.class, propertiesMap)
		.createStreamTableSource(propertiesMap);

	assertEquals(expected, actualSource);

	// test Kafka consumer
	final KafkaTableSourceBase actualKafkaSource = (KafkaTableSourceBase) actualSource;
	final StreamExecutionEnvironmentMock mock = new StreamExecutionEnvironmentMock();
	actualKafkaSource.getDataStream(mock);
	assertTrue(getExpectedFlinkKafkaConsumer().isAssignableFrom(mock.sourceFunction.getClass()));
}
 

@Test
@SuppressWarnings("unchecked")
public void testTableSource() {

	// prepare parameters for Kafka table source

	final TableSchema schema = TableSchema.builder()
		.field(FRUIT_NAME, Types.STRING())
		.field(COUNT, Types.DECIMAL())
		.field(EVENT_TIME, Types.SQL_TIMESTAMP())
		.field(PROC_TIME, Types.SQL_TIMESTAMP())
		.build();

	final List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors = Collections.singletonList(
		new RowtimeAttributeDescriptor(EVENT_TIME, new ExistingField(TIME), new AscendingTimestamps()));

	final Map<String, String> fieldMapping = new HashMap<>();
	fieldMapping.put(FRUIT_NAME, NAME);
	fieldMapping.put(NAME, NAME);
	fieldMapping.put(COUNT, COUNT);
	fieldMapping.put(TIME, TIME);

	final Map<KafkaTopicPartition, Long> specificOffsets = new HashMap<>();
	specificOffsets.put(new KafkaTopicPartition(TOPIC, PARTITION_0), OFFSET_0);
	specificOffsets.put(new KafkaTopicPartition(TOPIC, PARTITION_1), OFFSET_1);

	final TestDeserializationSchema deserializationSchema = new TestDeserializationSchema(
		TableSchema.builder()
			.field(NAME, Types.STRING())
			.field(COUNT, Types.DECIMAL())
			.field(TIME, Types.SQL_TIMESTAMP())
			.build()
			.toRowType()
	);

	final KafkaTableSourceBase expected = getExpectedKafkaTableSource(
		schema,
		Optional.of(PROC_TIME),
		rowtimeAttributeDescriptors,
		fieldMapping,
		TOPIC,
		KAFKA_PROPERTIES,
		deserializationSchema,
		StartupMode.SPECIFIC_OFFSETS,
		specificOffsets);

	TableSourceValidation.validateTableSource(expected);

	// construct table source using descriptors and table source factory

	final TestTableDescriptor testDesc = new TestTableDescriptor(
			new Kafka()
				.version(getKafkaVersion())
				.topic(TOPIC)
				.properties(KAFKA_PROPERTIES)
				.sinkPartitionerRoundRobin() // test if accepted although not needed
				.startFromSpecificOffsets(OFFSETS))
		.withFormat(new TestTableFormat())
		.withSchema(
			new Schema()
				.field(FRUIT_NAME, Types.STRING()).from(NAME)
				.field(COUNT, Types.DECIMAL()) // no from so it must match with the input
				.field(EVENT_TIME, Types.SQL_TIMESTAMP()).rowtime(
					new Rowtime().timestampsFromField(TIME).watermarksPeriodicAscending())
				.field(PROC_TIME, Types.SQL_TIMESTAMP()).proctime())
		.inAppendMode();

	final Map<String, String> propertiesMap = testDesc.toProperties();
	final TableSource<?> actualSource = TableFactoryService.find(StreamTableSourceFactory.class, propertiesMap)
		.createStreamTableSource(propertiesMap);

	assertEquals(expected, actualSource);

	// test Kafka consumer
	final KafkaTableSourceBase actualKafkaSource = (KafkaTableSourceBase) actualSource;
	final StreamExecutionEnvironmentMock mock = new StreamExecutionEnvironmentMock();
	actualKafkaSource.getDataStream(mock);
	assertTrue(getExpectedFlinkKafkaConsumer().isAssignableFrom(mock.sourceFunction.getClass()));
}
 

@Override
public void handleRequest() {

    TableSchema tableSchema = TripRecord.getTableSchema();

    FlinkPravegaJsonTableSource source = FlinkPravegaJsonTableSource.builder()
            .forStream(Stream.of(getScope(), getStream()).getScopedName())
            .withPravegaConfig(getPravegaConfig())
            .failOnMissingField(true)
            .withRowtimeAttribute("pickupTime",
                    new ExistingField("pickupTime"),
                    new BoundedOutOfOrderTimestamps(30000L))
            .withSchema(tableSchema)
            .withReaderGroupScope(getScope())
            .build();


    StreamExecutionEnvironment env = getStreamExecutionEnvironment();

    // create a TableEnvironment
    StreamTableEnvironment tEnv = StreamTableEnvironment.create(env);
    tEnv.registerTableSource("TaxiRide", source);

    String query =
            "SELECT " +
                    "destLocationId, wstart, wend, cnt " +
                    "FROM " +
                    "(SELECT " +
                    "destLocationId, " +
                    "HOP_START(pickupTime, INTERVAL '5' MINUTE, INTERVAL '15' MINUTE) AS wstart, " +
                    "HOP_END(pickupTime, INTERVAL '5' MINUTE, INTERVAL '15' MINUTE) AS wend, " +
                    "COUNT(destLocationId) AS cnt " +
                    "FROM " +
                    "(SELECT " +
                    "pickupTime, " +
                    "destLocationId " +
                    "FROM TaxiRide) " +
                    "GROUP BY destLocationId, HOP(pickupTime, INTERVAL '5' MINUTE, INTERVAL '15' MINUTE)) " +
                    "WHERE cnt > " + getLimit();

    Table results = tEnv.sqlQuery(query);

    tEnv.toAppendStream(results, Row.class).print();

    try {
        env.execute("Popular-Destination");
    } catch (Exception e) {
        log.error("Application Failed", e);
    }
}
 

@Test
@SuppressWarnings("unchecked")
public void testTableSource() {
	// prepare parameters for Kafka table source
	final TableSchema schema = TableSchema.builder()
		.field(FRUIT_NAME, DataTypes.STRING())
		.field(COUNT, DataTypes.DECIMAL(38, 18))
		.field(EVENT_TIME, DataTypes.TIMESTAMP(3))
		.field(PROC_TIME, DataTypes.TIMESTAMP(3))
		.build();

	final List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors = Collections.singletonList(
		new RowtimeAttributeDescriptor(EVENT_TIME, new ExistingField(TIME), new AscendingTimestamps()));

	final Map<String, String> fieldMapping = new HashMap<>();
	fieldMapping.put(FRUIT_NAME, NAME);
	fieldMapping.put(NAME, NAME);
	fieldMapping.put(COUNT, COUNT);
	fieldMapping.put(TIME, TIME);

	final Map<KafkaTopicPartition, Long> specificOffsets = new HashMap<>();
	specificOffsets.put(new KafkaTopicPartition(TOPIC, PARTITION_0), OFFSET_0);
	specificOffsets.put(new KafkaTopicPartition(TOPIC, PARTITION_1), OFFSET_1);

	final TestDeserializationSchema deserializationSchema = new TestDeserializationSchema(
		TableSchema.builder()
			.field(NAME, DataTypes.STRING())
			.field(COUNT, DataTypes.DECIMAL(38, 18))
			.field(TIME, DataTypes.TIMESTAMP(3))
			.build().toRowType()
	);

	final KafkaTableSourceBase expected = getExpectedKafkaTableSource(
		schema,
		Optional.of(PROC_TIME),
		rowtimeAttributeDescriptors,
		fieldMapping,
		TOPIC,
		KAFKA_PROPERTIES,
		deserializationSchema,
		StartupMode.SPECIFIC_OFFSETS,
		specificOffsets,
		0L);

	TableSourceValidation.validateTableSource(expected, schema);

	// construct table source using descriptors and table source factory
	final Map<String, String> propertiesMap = new HashMap<>();
	propertiesMap.putAll(createKafkaSourceProperties());
	propertiesMap.put("schema.watermark.0.rowtime", EVENT_TIME);
	propertiesMap.put("schema.watermark.0.strategy.expr", WATERMARK_EXPRESSION);
	propertiesMap.put("schema.watermark.0.strategy.data-type", WATERMARK_DATATYPE.toString());
	propertiesMap.put("schema.4.name", COMPUTED_COLUMN_NAME);
	propertiesMap.put("schema.4.data-type", COMPUTED_COLUMN_DATATYPE.toString());
	propertiesMap.put("schema.4.expr", COMPUTED_COLUMN_EXPRESSION);

	final TableSource<?> actualSource = TableFactoryService.find(StreamTableSourceFactory.class, propertiesMap)
		.createStreamTableSource(propertiesMap);

	assertEquals(expected, actualSource);

	// test Kafka consumer
	final KafkaTableSourceBase actualKafkaSource = (KafkaTableSourceBase) actualSource;
	final StreamExecutionEnvironmentMock mock = new StreamExecutionEnvironmentMock();
	actualKafkaSource.getDataStream(mock);
	assertTrue(getExpectedFlinkKafkaConsumer().isAssignableFrom(mock.sourceFunction.getClass()));
	// Test commitOnCheckpoints flag should be true when set consumer group.
	assertTrue(((FlinkKafkaConsumerBase) mock.sourceFunction).getEnableCommitOnCheckpoints());
}
 

@Test
@SuppressWarnings("unchecked")
public void testTableSourceWithLegacyProperties() {
	// prepare parameters for Kafka table source
	final TableSchema schema = TableSchema.builder()
		.field(FRUIT_NAME, DataTypes.STRING())
		.field(COUNT, DataTypes.DECIMAL(38, 18))
		.field(EVENT_TIME, DataTypes.TIMESTAMP(3))
		.field(PROC_TIME, DataTypes.TIMESTAMP(3))
		.build();

	final List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors = Collections.singletonList(
		new RowtimeAttributeDescriptor(EVENT_TIME, new ExistingField(TIME), new AscendingTimestamps()));

	final Map<String, String> fieldMapping = new HashMap<>();
	fieldMapping.put(FRUIT_NAME, NAME);
	fieldMapping.put(NAME, NAME);
	fieldMapping.put(COUNT, COUNT);
	fieldMapping.put(TIME, TIME);

	final Map<KafkaTopicPartition, Long> specificOffsets = new HashMap<>();
	specificOffsets.put(new KafkaTopicPartition(TOPIC, PARTITION_0), OFFSET_0);
	specificOffsets.put(new KafkaTopicPartition(TOPIC, PARTITION_1), OFFSET_1);

	final TestDeserializationSchema deserializationSchema = new TestDeserializationSchema(
		TableSchema.builder()
			.field(NAME, DataTypes.STRING())
			.field(COUNT, DataTypes.DECIMAL(38, 18))
			.field(TIME, DataTypes.TIMESTAMP(3))
			.build().toRowType()
	);

	final KafkaTableSourceBase expected = getExpectedKafkaTableSource(
		schema,
		Optional.of(PROC_TIME),
		rowtimeAttributeDescriptors,
		fieldMapping,
		TOPIC,
		KAFKA_PROPERTIES,
		deserializationSchema,
		StartupMode.SPECIFIC_OFFSETS,
		specificOffsets,
		0L);

	TableSourceValidation.validateTableSource(expected, schema);

	// construct table source using descriptors and table source factory
	final Map<String, String> legacyPropertiesMap = new HashMap<>();
	legacyPropertiesMap.putAll(createKafkaSourceProperties());

	// use legacy properties
	legacyPropertiesMap.remove("connector.specific-offsets");
	legacyPropertiesMap.remove("connector.properties.bootstrap.servers");
	legacyPropertiesMap.remove("connector.properties.group.id");

	// keep compatible with a specified update-mode
	legacyPropertiesMap.put("update-mode", "append");

	// legacy properties for specific-offsets and properties
	legacyPropertiesMap.put("connector.specific-offsets.0.partition", "0");
	legacyPropertiesMap.put("connector.specific-offsets.0.offset", "100");
	legacyPropertiesMap.put("connector.specific-offsets.1.partition", "1");
	legacyPropertiesMap.put("connector.specific-offsets.1.offset", "123");
	legacyPropertiesMap.put("connector.properties.0.key", "bootstrap.servers");
	legacyPropertiesMap.put("connector.properties.0.value", "dummy");
	legacyPropertiesMap.put("connector.properties.1.key", "group.id");
	legacyPropertiesMap.put("connector.properties.1.value", "dummy");

	final TableSource<?> actualSource = TableFactoryService.find(StreamTableSourceFactory.class, legacyPropertiesMap)
		.createStreamTableSource(legacyPropertiesMap);

	assertEquals(expected, actualSource);

	// test Kafka consumer
	final KafkaTableSourceBase actualKafkaSource = (KafkaTableSourceBase) actualSource;
	final StreamExecutionEnvironmentMock mock = new StreamExecutionEnvironmentMock();
	actualKafkaSource.getDataStream(mock);
	assertTrue(getExpectedFlinkKafkaConsumer().isAssignableFrom(mock.sourceFunction.getClass()));
}