org.apache.flink.table.descriptors.Rowtime Java Examples
The following examples show how to use
org.apache.flink.table.descriptors.Rowtime.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: FlinkPravegaTableFactoryTest.java From flink-connectors with Apache License 2.0 | 6 votes |
/** * Rowtime attribute should be of type TIMESTAMP. */ @Test (expected = ValidationException.class) public void testWrongRowTimeAttributeType() { final Schema schema = new Schema() .field("name", DataTypes.STRING()) .field("age", DataTypes.INT()).rowtime(new Rowtime() .timestampsFromField("age") .watermarksFromStrategy( new BoundedOutOfOrderTimestamps(30000L))); Pravega pravega = new Pravega(); Stream stream = Stream.of(SCOPE, STREAM); pravega.tableSourceReaderBuilder() .forStream(stream) .withPravegaConfig(PRAVEGA_CONFIG); final TestTableDescriptor testDesc = new TestTableDescriptor(pravega) .withFormat(JSON) .withSchema(schema) .inAppendMode(); final Map<String, String> propertiesMap = testDesc.toProperties(); FlinkPravegaTableFactoryBase tableFactoryBase = new FlinkPravegaStreamTableSourceFactory(); tableFactoryBase.createFlinkPravegaTableSource(propertiesMap); fail("Schema validation failed"); }
Example #2
Source File: KafkaTableSourceSinkFactoryTestBase.java From flink with Apache License 2.0 | 6 votes |
protected Map<String, String> createKafkaSourceProperties() { return new TestTableDescriptor( new Kafka() .version(getKafkaVersion()) .topic(TOPIC) .properties(KAFKA_PROPERTIES) .sinkPartitionerRoundRobin() // test if accepted although not needed .startFromSpecificOffsets(OFFSETS)) .withFormat(new TestTableFormat()) .withSchema( new Schema() .field(FRUIT_NAME, DataTypes.STRING()).from(NAME) .field(COUNT, DataTypes.DECIMAL(38, 18)) // no from so it must match with the input .field(EVENT_TIME, DataTypes.TIMESTAMP(3)).rowtime( new Rowtime().timestampsFromField(TIME).watermarksPeriodicAscending()) .field(PROC_TIME, DataTypes.TIMESTAMP(3)).proctime()) .toProperties(); }
Example #3
Source File: BoundedOutOfOrderTimestamps.java From flink with Apache License 2.0 | 5 votes |
@Override public Map<String, String> toProperties() { Map<String, String> map = new HashMap<>(); map.put( Rowtime.ROWTIME_WATERMARKS_TYPE, Rowtime.ROWTIME_WATERMARKS_TYPE_VALUE_PERIODIC_BOUNDED); map.put(Rowtime.ROWTIME_WATERMARKS_DELAY, String.valueOf(delay)); return map; }
Example #4
Source File: TimestampExtractor.java From flink with Apache License 2.0 | 5 votes |
/** * This method is a default implementation that uses java serialization and it is discouraged. * All implementation should provide a more specific set of properties. */ @Override public Map<String, String> toProperties() { Map<String, String> properties = new HashMap<>(); properties.put(Rowtime.ROWTIME_TIMESTAMPS_TYPE, Rowtime.ROWTIME_TIMESTAMPS_TYPE_VALUE_CUSTOM); properties.put(Rowtime.ROWTIME_TIMESTAMPS_CLASS, this.getClass().getName()); properties.put(Rowtime.ROWTIME_TIMESTAMPS_SERIALIZED, EncodingUtils.encodeObjectToString(this)); return properties; }
Example #5
Source File: WatermarkStrategy.java From flink with Apache License 2.0 | 5 votes |
/** * This method is a default implementation that uses java serialization and it is discouraged. * All implementation should provide a more specific set of properties. */ @Override public Map<String, String> toProperties() { Map<String, String> properties = new HashMap<>(); properties.put(Rowtime.ROWTIME_WATERMARKS_TYPE, Rowtime.ROWTIME_WATERMARKS_TYPE_VALUE_CUSTOM); properties.put(Rowtime.ROWTIME_WATERMARKS_CLASS, this.getClass().getName()); properties.put(Rowtime.ROWTIME_WATERMARKS_SERIALIZED, EncodingUtils.encodeObjectToString(this)); return properties; }
Example #6
Source File: AscendingTimestamps.java From flink with Apache License 2.0 | 5 votes |
@Override public Map<String, String> toProperties() { Map<String, String> map = new HashMap<>(); map.put( Rowtime.ROWTIME_WATERMARKS_TYPE, Rowtime.ROWTIME_WATERMARKS_TYPE_VALUE_PERIODIC_ASCENDING); return map; }
Example #7
Source File: ExistingField.java From flink with Apache License 2.0 | 5 votes |
@Override public Map<String, String> toProperties() { Map<String, String> map = new HashMap<>(); map.put(Rowtime.ROWTIME_TIMESTAMPS_TYPE, Rowtime.ROWTIME_TIMESTAMPS_TYPE_VALUE_FROM_FIELD); map.put(Rowtime.ROWTIME_TIMESTAMPS_FROM, field); return map; }
Example #8
Source File: ExistingField.java From flink with Apache License 2.0 | 5 votes |
@Override public Map<String, String> toProperties() { Map<String, String> map = new HashMap<>(); map.put(Rowtime.ROWTIME_TIMESTAMPS_TYPE, Rowtime.ROWTIME_TIMESTAMPS_TYPE_VALUE_FROM_FIELD); map.put(Rowtime.ROWTIME_TIMESTAMPS_FROM, field); return map; }
Example #9
Source File: AscendingTimestamps.java From flink with Apache License 2.0 | 5 votes |
@Override public Map<String, String> toProperties() { Map<String, String> map = new HashMap<>(); map.put( Rowtime.ROWTIME_WATERMARKS_TYPE, Rowtime.ROWTIME_WATERMARKS_TYPE_VALUE_PERIODIC_ASCENDING); return map; }
Example #10
Source File: BoundedOutOfOrderTimestamps.java From flink with Apache License 2.0 | 5 votes |
@Override public Map<String, String> toProperties() { Map<String, String> map = new HashMap<>(); map.put( Rowtime.ROWTIME_WATERMARKS_TYPE, Rowtime.ROWTIME_WATERMARKS_TYPE_VALUE_PERIODIC_BOUNDED); map.put(Rowtime.ROWTIME_WATERMARKS_DELAY, String.valueOf(delay)); return map; }
Example #11
Source File: WatermarkStrategy.java From flink with Apache License 2.0 | 5 votes |
/** * This method is a default implementation that uses java serialization and it is discouraged. * All implementation should provide a more specific set of properties. */ @Override public Map<String, String> toProperties() { Map<String, String> properties = new HashMap<>(); properties.put(Rowtime.ROWTIME_WATERMARKS_TYPE, Rowtime.ROWTIME_WATERMARKS_TYPE_VALUE_CUSTOM); properties.put(Rowtime.ROWTIME_WATERMARKS_CLASS, this.getClass().getName()); properties.put(Rowtime.ROWTIME_WATERMARKS_SERIALIZED, EncodingUtils.encodeObjectToString(this)); return properties; }
Example #12
Source File: TimestampExtractor.java From flink with Apache License 2.0 | 5 votes |
/** * This method is a default implementation that uses java serialization and it is discouraged. * All implementation should provide a more specific set of properties. */ @Override public Map<String, String> toProperties() { Map<String, String> properties = new HashMap<>(); properties.put(Rowtime.ROWTIME_TIMESTAMPS_TYPE, Rowtime.ROWTIME_TIMESTAMPS_TYPE_VALUE_CUSTOM); properties.put(Rowtime.ROWTIME_TIMESTAMPS_CLASS, this.getClass().getName()); properties.put(Rowtime.ROWTIME_TIMESTAMPS_SERIALIZED, EncodingUtils.encodeObjectToString(this)); return properties; }
Example #13
Source File: FlinkPravegaTableSourceTest.java From flink-connectors with Apache License 2.0 | 4 votes |
@Test @SuppressWarnings("unchecked") public void testTableSourceDescriptorWithWatermark() { final String cityName = "fruitName"; final String total = "count"; final String eventTime = "eventTime"; final String controllerUri = "tcp://localhost:9090"; final String streamName = "test"; final String scopeName = "test"; Stream stream = Stream.of(scopeName, streamName); PravegaConfig pravegaConfig = PravegaConfig.fromDefaults() .withControllerURI(URI.create(controllerUri)) .withDefaultScope(scopeName); // construct table source using descriptors and table source factory Pravega pravega = new Pravega(); pravega.tableSourceReaderBuilder() .withTimestampAssigner(new MyAssigner()) .withReaderGroupScope(stream.getScope()) .forStream(stream) .withPravegaConfig(pravegaConfig); final TableSchema tableSchema = TableSchema.builder() .field(cityName, DataTypes.STRING()) .field(total, DataTypes.INT()) .field(eventTime, DataTypes.TIMESTAMP(3)) .build(); final TestTableDescriptor testDesc = new TestTableDescriptor(pravega) .withFormat(new Json().failOnMissingField(false)) .withSchema( new Schema() .field(cityName, DataTypes.STRING()) .field(total, DataTypes.INT()) .field(eventTime, DataTypes.TIMESTAMP(3)) .rowtime(new Rowtime() .timestampsFromSource() .watermarksFromSource() )) .inAppendMode(); final Map<String, String> propertiesMap = testDesc.toProperties(); final TableSource<?> actualSource = TableFactoryService.find(StreamTableSourceFactory.class, propertiesMap) .createStreamTableSource(propertiesMap); assertNotNull(actualSource); TableSourceValidation.validateTableSource(actualSource, tableSchema); }
Example #14
Source File: StreamRecordTimestamp.java From flink with Apache License 2.0 | 4 votes |
@Override public Map<String, String> toProperties() { Map<String, String> map = new HashMap<>(); map.put(Rowtime.ROWTIME_TIMESTAMPS_TYPE, Rowtime.ROWTIME_TIMESTAMPS_TYPE_VALUE_FROM_SOURCE); return map; }
Example #15
Source File: PreserveWatermarks.java From flink with Apache License 2.0 | 4 votes |
@Override public Map<String, String> toProperties() { Map<String, String> map = new HashMap<>(); map.put(Rowtime.ROWTIME_WATERMARKS_TYPE, Rowtime.ROWTIME_WATERMARKS_TYPE_VALUE_FROM_SOURCE); return map; }
Example #16
Source File: FlinkPravegaTableSinkTest.java From flink-connectors with Apache License 2.0 | 4 votes |
@Test @SuppressWarnings("unchecked") public void testTableSinkDescriptor() { final String cityName = "fruitName"; final String total = "count"; final String eventTime = "eventTime"; final String procTime = "procTime"; final String controllerUri = "tcp://localhost:9090"; final long delay = 3000L; final String streamName = "test"; final String scopeName = "test"; Stream stream = Stream.of(scopeName, streamName); PravegaConfig pravegaConfig = PravegaConfig.fromDefaults() .withControllerURI(URI.create(controllerUri)) .withDefaultScope(scopeName); // construct table sink using descriptors and table sink factory Pravega pravega = new Pravega(); pravega.tableSinkWriterBuilder() .withRoutingKeyField(cityName) .withWriterMode(PravegaWriterMode.EXACTLY_ONCE) .enableWatermark(true) .forStream(stream) .enableMetrics(true) .withPravegaConfig(pravegaConfig); final FlinkPravegaTableSourceTest.TestTableDescriptor testDesc = new FlinkPravegaTableSourceTest.TestTableDescriptor(pravega) .withFormat(new Json().failOnMissingField(false)) .withSchema( new Schema() .field(cityName, DataTypes.STRING()) .field(total, DataTypes.BIGINT()) .field(eventTime, DataTypes.TIMESTAMP(3)) .rowtime(new Rowtime() .timestampsFromField(eventTime) .watermarksFromStrategy(new BoundedOutOfOrderTimestamps(delay)) ) .field(procTime, DataTypes.TIMESTAMP(3)).proctime() ) .inAppendMode(); final Map<String, String> propertiesMap = testDesc.toProperties(); final TableSink<?> sink = TableFactoryService.find(StreamTableSinkFactory.class, propertiesMap) .createStreamTableSink(propertiesMap); assertNotNull(sink); }
Example #17
Source File: KafkaTableSourceSinkFactoryTestBase.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
@Test @SuppressWarnings("unchecked") public void testTableSource() { // prepare parameters for Kafka table source final TableSchema schema = TableSchema.builder() .field(FRUIT_NAME, Types.STRING()) .field(COUNT, Types.DECIMAL()) .field(EVENT_TIME, Types.SQL_TIMESTAMP()) .field(PROC_TIME, Types.SQL_TIMESTAMP()) .build(); final List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors = Collections.singletonList( new RowtimeAttributeDescriptor(EVENT_TIME, new ExistingField(TIME), new AscendingTimestamps())); final Map<String, String> fieldMapping = new HashMap<>(); fieldMapping.put(FRUIT_NAME, NAME); fieldMapping.put(NAME, NAME); fieldMapping.put(COUNT, COUNT); fieldMapping.put(TIME, TIME); final Map<KafkaTopicPartition, Long> specificOffsets = new HashMap<>(); specificOffsets.put(new KafkaTopicPartition(TOPIC, PARTITION_0), OFFSET_0); specificOffsets.put(new KafkaTopicPartition(TOPIC, PARTITION_1), OFFSET_1); final TestDeserializationSchema deserializationSchema = new TestDeserializationSchema( TableSchema.builder() .field(NAME, Types.STRING()) .field(COUNT, Types.DECIMAL()) .field(TIME, Types.SQL_TIMESTAMP()) .build() .toRowType() ); final KafkaTableSourceBase expected = getExpectedKafkaTableSource( schema, Optional.of(PROC_TIME), rowtimeAttributeDescriptors, fieldMapping, TOPIC, KAFKA_PROPERTIES, deserializationSchema, StartupMode.SPECIFIC_OFFSETS, specificOffsets); TableSourceUtil.validateTableSource(expected); // construct table source using descriptors and table source factory final TestTableDescriptor testDesc = new TestTableDescriptor( new Kafka() .version(getKafkaVersion()) .topic(TOPIC) .properties(KAFKA_PROPERTIES) .sinkPartitionerRoundRobin() // test if accepted although not needed .startFromSpecificOffsets(OFFSETS)) .withFormat(new TestTableFormat()) .withSchema( new Schema() .field(FRUIT_NAME, Types.STRING()).from(NAME) .field(COUNT, Types.DECIMAL()) // no from so it must match with the input .field(EVENT_TIME, Types.SQL_TIMESTAMP()).rowtime( new Rowtime().timestampsFromField(TIME).watermarksPeriodicAscending()) .field(PROC_TIME, Types.SQL_TIMESTAMP()).proctime()) .inAppendMode(); final Map<String, String> propertiesMap = testDesc.toProperties(); final TableSource<?> actualSource = TableFactoryService.find(StreamTableSourceFactory.class, propertiesMap) .createStreamTableSource(propertiesMap); assertEquals(expected, actualSource); // test Kafka consumer final KafkaTableSourceBase actualKafkaSource = (KafkaTableSourceBase) actualSource; final StreamExecutionEnvironmentMock mock = new StreamExecutionEnvironmentMock(); actualKafkaSource.getDataStream(mock); assertTrue(getExpectedFlinkKafkaConsumer().isAssignableFrom(mock.sourceFunction.getClass())); }
Example #18
Source File: FlinkPravegaTableSourceTest.java From flink-connectors with Apache License 2.0 | 4 votes |
@Test @SuppressWarnings("unchecked") public void testTableSourceDescriptor() { final String cityName = "fruitName"; final String total = "count"; final String eventTime = "eventTime"; final String procTime = "procTime"; final String controllerUri = "tcp://localhost:9090"; final long delay = 3000L; final String streamName = "test"; final String scopeName = "test"; final TableSchema tableSchema = TableSchema.builder() .field(cityName, DataTypes.STRING()) .field(total, DataTypes.BIGINT()) .field(eventTime, DataTypes.TIMESTAMP(3)) .field(procTime, DataTypes.TIMESTAMP(3)) .build(); Stream stream = Stream.of(scopeName, streamName); PravegaConfig pravegaConfig = PravegaConfig.fromDefaults() .withControllerURI(URI.create(controllerUri)) .withDefaultScope(scopeName); // construct table source using descriptors and table source factory Pravega pravega = new Pravega(); pravega.tableSourceReaderBuilder() .withReaderGroupScope(stream.getScope()) .forStream(stream) .withPravegaConfig(pravegaConfig); final TestTableDescriptor testDesc = new TestTableDescriptor(pravega) .withFormat(new Json().failOnMissingField(false)) .withSchema( new Schema() .field(cityName, DataTypes.STRING()) .field(total, DataTypes.BIGINT()) .field(eventTime, DataTypes.TIMESTAMP(3)) .rowtime(new Rowtime() .timestampsFromField(eventTime) .watermarksFromStrategy(new BoundedOutOfOrderTimestamps(delay)) ) .field(procTime, DataTypes.TIMESTAMP(3)).proctime()) .inAppendMode(); final Map<String, String> propertiesMap = testDesc.toProperties(); final TableSource<?> actualSource = TableFactoryService.find(StreamTableSourceFactory.class, propertiesMap) .createStreamTableSource(propertiesMap); assertNotNull(actualSource); TableSourceValidation.validateTableSource(actualSource, tableSchema); }
Example #19
Source File: FlinkPravegaTableITCase.java From flink-connectors with Apache License 2.0 | 4 votes |
private void testTableSourceStreamingDescriptor(Stream stream, PravegaConfig pravegaConfig) throws Exception { final StreamExecutionEnvironment execEnvRead = StreamExecutionEnvironment.getExecutionEnvironment(); execEnvRead.setParallelism(1); execEnvRead.enableCheckpointing(100); execEnvRead.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); StreamTableEnvironment tableEnv = StreamTableEnvironment.create(execEnvRead, EnvironmentSettings.newInstance() // watermark is only supported in blink planner .useBlinkPlanner() .inStreamingMode() .build()); RESULTS.clear(); // read data from the stream using Table reader Schema schema = new Schema() .field("user", DataTypes.STRING()) .field("uri", DataTypes.STRING()) .field("accessTime", DataTypes.TIMESTAMP(3)).rowtime( new Rowtime().timestampsFromField("accessTime").watermarksPeriodicBounded(30000L)); Pravega pravega = new Pravega(); pravega.tableSourceReaderBuilder() .withReaderGroupScope(stream.getScope()) .forStream(stream) .withPravegaConfig(pravegaConfig); ConnectTableDescriptor desc = tableEnv.connect(pravega) .withFormat(new Json().failOnMissingField(true)) .withSchema(schema) .inAppendMode(); final Map<String, String> propertiesMap = desc.toProperties(); final TableSource<?> source = TableFactoryService.find(StreamTableSourceFactory.class, propertiesMap) .createStreamTableSource(propertiesMap); String tableSourcePath = tableEnv.getCurrentDatabase() + "." + "MyTableRow"; ConnectorCatalogTable<?, ?> connectorCatalogSourceTable = ConnectorCatalogTable.source(source, false); tableEnv.getCatalog(tableEnv.getCurrentCatalog()).get().createTable( ObjectPath.fromString(tableSourcePath), connectorCatalogSourceTable, false); String sqlQuery = "SELECT user, " + "TUMBLE_END(accessTime, INTERVAL '5' MINUTE) AS accessTime, " + "COUNT(uri) AS cnt " + "from MyTableRow GROUP BY " + "user, TUMBLE(accessTime, INTERVAL '5' MINUTE)"; Table result = tableEnv.sqlQuery(sqlQuery); DataStream<Tuple2<Boolean, Row>> resultSet = tableEnv.toRetractStream(result, Row.class); StringSink2 stringSink = new StringSink2(8); resultSet.addSink(stringSink); try { execEnvRead.execute("ReadRowData"); } catch (Exception e) { if (!(ExceptionUtils.getRootCause(e) instanceof SuccessException)) { throw e; } } log.info("results: {}", RESULTS); boolean compare = compare(RESULTS, getExpectedResultsAppend()); assertTrue("Output does not match expected result", compare); }
Example #20
Source File: StreamRecordTimestamp.java From flink with Apache License 2.0 | 4 votes |
@Override public Map<String, String> toProperties() { Map<String, String> map = new HashMap<>(); map.put(Rowtime.ROWTIME_TIMESTAMPS_TYPE, Rowtime.ROWTIME_TIMESTAMPS_TYPE_VALUE_FROM_SOURCE); return map; }
Example #21
Source File: PreserveWatermarks.java From flink with Apache License 2.0 | 4 votes |
@Override public Map<String, String> toProperties() { Map<String, String> map = new HashMap<>(); map.put(Rowtime.ROWTIME_WATERMARKS_TYPE, Rowtime.ROWTIME_WATERMARKS_TYPE_VALUE_FROM_SOURCE); return map; }
Example #22
Source File: KafkaTableSourceSinkFactoryTestBase.java From flink with Apache License 2.0 | 4 votes |
@Test @SuppressWarnings("unchecked") public void testTableSource() { // prepare parameters for Kafka table source final TableSchema schema = TableSchema.builder() .field(FRUIT_NAME, Types.STRING()) .field(COUNT, Types.DECIMAL()) .field(EVENT_TIME, Types.SQL_TIMESTAMP()) .field(PROC_TIME, Types.SQL_TIMESTAMP()) .build(); final List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors = Collections.singletonList( new RowtimeAttributeDescriptor(EVENT_TIME, new ExistingField(TIME), new AscendingTimestamps())); final Map<String, String> fieldMapping = new HashMap<>(); fieldMapping.put(FRUIT_NAME, NAME); fieldMapping.put(NAME, NAME); fieldMapping.put(COUNT, COUNT); fieldMapping.put(TIME, TIME); final Map<KafkaTopicPartition, Long> specificOffsets = new HashMap<>(); specificOffsets.put(new KafkaTopicPartition(TOPIC, PARTITION_0), OFFSET_0); specificOffsets.put(new KafkaTopicPartition(TOPIC, PARTITION_1), OFFSET_1); final TestDeserializationSchema deserializationSchema = new TestDeserializationSchema( TableSchema.builder() .field(NAME, Types.STRING()) .field(COUNT, Types.DECIMAL()) .field(TIME, Types.SQL_TIMESTAMP()) .build() .toRowType() ); final KafkaTableSourceBase expected = getExpectedKafkaTableSource( schema, Optional.of(PROC_TIME), rowtimeAttributeDescriptors, fieldMapping, TOPIC, KAFKA_PROPERTIES, deserializationSchema, StartupMode.SPECIFIC_OFFSETS, specificOffsets); TableSourceValidation.validateTableSource(expected); // construct table source using descriptors and table source factory final TestTableDescriptor testDesc = new TestTableDescriptor( new Kafka() .version(getKafkaVersion()) .topic(TOPIC) .properties(KAFKA_PROPERTIES) .sinkPartitionerRoundRobin() // test if accepted although not needed .startFromSpecificOffsets(OFFSETS)) .withFormat(new TestTableFormat()) .withSchema( new Schema() .field(FRUIT_NAME, Types.STRING()).from(NAME) .field(COUNT, Types.DECIMAL()) // no from so it must match with the input .field(EVENT_TIME, Types.SQL_TIMESTAMP()).rowtime( new Rowtime().timestampsFromField(TIME).watermarksPeriodicAscending()) .field(PROC_TIME, Types.SQL_TIMESTAMP()).proctime()) .inAppendMode(); final Map<String, String> propertiesMap = testDesc.toProperties(); final TableSource<?> actualSource = TableFactoryService.find(StreamTableSourceFactory.class, propertiesMap) .createStreamTableSource(propertiesMap); assertEquals(expected, actualSource); // test Kafka consumer final KafkaTableSourceBase actualKafkaSource = (KafkaTableSourceBase) actualSource; final StreamExecutionEnvironmentMock mock = new StreamExecutionEnvironmentMock(); actualKafkaSource.getDataStream(mock); assertTrue(getExpectedFlinkKafkaConsumer().isAssignableFrom(mock.sourceFunction.getClass())); }