org.apache.flink.streaming.api.functions.timestamps.AscendingTimestampExtractor Java Examples

public static void main(String[] args) throws Exception {

        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

        // 指定系统时间概念为 event time
        env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);

        List<Tuple2<String, Long>> collectionInput = new ArrayList<>();
        Tuple2<String, Long> a = new Tuple2<>("first event", 1L);
        Tuple2<String, Long> b = new Tuple2<>("second event", 2L);
        collectionInput.add(a);
        collectionInput.add(b);

        // 使用 Ascending 分配 时间信息和 watermark
        DataStream<Tuple2<String, Long>> text = env.fromCollection(collectionInput);
        text.assignTimestampsAndWatermarks(new AscendingTimestampExtractor<Tuple2<String, Long>>() {
            @Override
            public long extractAscendingTimestamp(Tuple2<String, Long> element) {
                return element.f1;
            }
        });

        env.execute();
    }
 

@Test
public void testUnboundedPojoStreamAndReturnPojo() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStream<Event> input = env.addSource(new RandomEventSource(5));
    input.assignTimestampsAndWatermarks(new AscendingTimestampExtractor<Event>() {
        @Override
        public long extractAscendingTimestamp(Event element) {
            return element.getTimestamp();
        }
    });

    DataStream<Event> output = SiddhiCEP
        .define("inputStream", input, "id", "name", "price", "timestamp")
        .cql("from inputStream select timestamp, id, name, price insert into  outputStream")
        .returns("outputStream", Event.class);

    String resultPath = tempFolder.newFile().toURI().toString();
    output.writeAsText(resultPath, FileSystem.WriteMode.OVERWRITE);
    env.execute();
    assertEquals(5, getLineCount(resultPath));
}
 

@Test
public void testUnboundedPojoStreamAndReturnPojo() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStream<Event> input = env.addSource(new RandomEventSource(5));
    input.assignTimestampsAndWatermarks(new AscendingTimestampExtractor<Event>() {
        @Override
        public long extractAscendingTimestamp(Event element) {
            return element.getTimestamp();
        }
    });

    DataStream<Event> output = SiddhiCEP
        .define("inputStream", input, "id", "name", "price", "timestamp")
        .cql("from inputStream select timestamp, id, name, price insert into  outputStream")
        .returns("outputStream", Event.class);

    String resultPath = tempFolder.newFile().toURI().toString();
    output.writeAsText(resultPath, FileSystem.WriteMode.OVERWRITE);
    env.execute();
    assertEquals(5, getLineCount(resultPath));
}
 

private void runValidTests(AscendingTimestampExtractor<Long> extractor) {
	assertEquals(13L, extractor.extractTimestamp(13L, -1L));
	assertEquals(13L, extractor.extractTimestamp(13L, 0L));
	assertEquals(14L, extractor.extractTimestamp(14L, 0L));
	assertEquals(20L, extractor.extractTimestamp(20L, 0L));
	assertEquals(20L, extractor.extractTimestamp(20L, 0L));
	assertEquals(20L, extractor.extractTimestamp(20L, 0L));
	assertEquals(500L, extractor.extractTimestamp(500L, 0L));

	assertEquals(Long.MAX_VALUE - 1, extractor.extractTimestamp(Long.MAX_VALUE - 1, 99999L));
}
 

private void runInvalidTest(AscendingTimestampExtractor<Long> extractor) {
	assertEquals(1000L, extractor.extractTimestamp(1000L, 100));
	assertEquals(1000L, extractor.extractTimestamp(1000L, 100));

	// violation
	assertEquals(999L, extractor.extractTimestamp(999L, 100));
}
 

private void runValidTests(AscendingTimestampExtractor<Long> extractor) {
	assertEquals(13L, extractor.extractTimestamp(13L, -1L));
	assertEquals(13L, extractor.extractTimestamp(13L, 0L));
	assertEquals(14L, extractor.extractTimestamp(14L, 0L));
	assertEquals(20L, extractor.extractTimestamp(20L, 0L));
	assertEquals(20L, extractor.extractTimestamp(20L, 0L));
	assertEquals(20L, extractor.extractTimestamp(20L, 0L));
	assertEquals(500L, extractor.extractTimestamp(500L, 0L));

	assertEquals(Long.MAX_VALUE - 1, extractor.extractTimestamp(Long.MAX_VALUE - 1, 99999L));
}
 

@Test
public void testInitialAndFinalWatermark() {
	AscendingTimestampExtractor<Long> extractor = new LongExtractor();
	assertEquals(Long.MIN_VALUE, extractor.getCurrentWatermark().getTimestamp());

	extractor.extractTimestamp(Long.MIN_VALUE, -1L);

	extractor.extractTimestamp(Long.MAX_VALUE, -1L);
	assertEquals(Long.MAX_VALUE - 1, extractor.getCurrentWatermark().getTimestamp());
}
 

@Test
public void testWithDefaultHandler() {
	AscendingTimestampExtractor<Long> extractor = new LongExtractor();

	runValidTests(extractor);
	runInvalidTest(extractor);
}
 

@Test
public void testWithLoggingHandler() {
	AscendingTimestampExtractor<Long> extractor = new LongExtractor()
			.withViolationHandler(new AscendingTimestampExtractor.LoggingHandler());

	runValidTests(extractor);
	runInvalidTest(extractor);
}
 

@Test
public void testWithIgnoringHandler() {
	AscendingTimestampExtractor<Long> extractor = new LongExtractor()
			.withViolationHandler(new AscendingTimestampExtractor.IgnoringHandler());

	runValidTests(extractor);
	runInvalidTest(extractor);
}
 

@Test
public void testWithFailingHandler() {
	AscendingTimestampExtractor<Long> extractor = new LongExtractor()
			.withViolationHandler(new AscendingTimestampExtractor.FailingHandler());

	runValidTests(extractor);
	try {
		runInvalidTest(extractor);
		fail("should fail with an exception");
	} catch (Exception ignored) {}
}
 

/**
 * Returns a DataStream of TaxiRide events from a CSV file.
 *
 * @param env The execution environment.
 * @param csvFile The path of the CSV file to read.
 * @return A DataStream of TaxiRide events.
 */
public static DataStream<TaxiRide> getRides(StreamExecutionEnvironment env, String csvFile) {

    // create input format to read the CSV file
    RowCsvInputFormat inputFormat = new RowCsvInputFormat(
            null, // input path is configured later
            inputFieldTypes,
            "\n",
            ",");

    // read file sequentially (with a parallelism of 1)
    DataStream<Row> parsedRows = env
            .readFile(inputFormat, csvFile)
            .returns(Types.ROW(inputFieldTypes))
            .setParallelism(1);

    // convert parsed CSV rows into TaxiRides, extract timestamps, and assign watermarks
    return parsedRows
            // map to TaxiRide POJOs
            .map(new RideMapper())
            // define drop-off time as event-time timestamps and generate ascending watermarks.
            .assignTimestampsAndWatermarks(new AscendingTimestampExtractor<TaxiRide>() {
                @Override
                public long extractAscendingTimestamp(TaxiRide ride) {
                    return ride.dropOffTime;
                }
            });
}
 

private void runInvalidTest(AscendingTimestampExtractor<Long> extractor) {
	assertEquals(1000L, extractor.extractTimestamp(1000L, 100));
	assertEquals(1000L, extractor.extractTimestamp(1000L, 100));

	// violation
	assertEquals(999L, extractor.extractTimestamp(999L, 100));
}
 

@Test
public void testInitialAndFinalWatermark() {
	AscendingTimestampExtractor<Long> extractor = new LongExtractor();
	assertEquals(Long.MIN_VALUE, extractor.getCurrentWatermark().getTimestamp());

	extractor.extractTimestamp(Long.MIN_VALUE, -1L);

	extractor.extractTimestamp(Long.MAX_VALUE, -1L);
	assertEquals(Long.MAX_VALUE - 1, extractor.getCurrentWatermark().getTimestamp());
}
 

@Test
public void testWithDefaultHandler() {
	AscendingTimestampExtractor<Long> extractor = new LongExtractor();

	runValidTests(extractor);
	runInvalidTest(extractor);
}
 

@Test
public void testWithIgnoringHandler() {
	AscendingTimestampExtractor<Long> extractor = new LongExtractor()
			.withViolationHandler(new AscendingTimestampExtractor.IgnoringHandler());

	runValidTests(extractor);
	runInvalidTest(extractor);
}
 

@Test
public void testWithFailingHandler() {
	AscendingTimestampExtractor<Long> extractor = new LongExtractor()
			.withViolationHandler(new AscendingTimestampExtractor.FailingHandler());

	runValidTests(extractor);
	try {
		runInvalidTest(extractor);
		fail("should fail with an exception");
	} catch (Exception ignored) {}
}
 

@Test
public void testWithIgnoringHandler() {
	AscendingTimestampExtractor<Long> extractor = new LongExtractor()
			.withViolationHandler(new AscendingTimestampExtractor.IgnoringHandler());

	runValidTests(extractor);
	runInvalidTest(extractor);
}
 

@Test
public void testWithLoggingHandler() {
	AscendingTimestampExtractor<Long> extractor = new LongExtractor()
			.withViolationHandler(new AscendingTimestampExtractor.LoggingHandler());

	runValidTests(extractor);
	runInvalidTest(extractor);
}
 

@Test
public void testWithDefaultHandler() {
	AscendingTimestampExtractor<Long> extractor = new LongExtractor();

	runValidTests(extractor);
	runInvalidTest(extractor);
}
 

@Test
public void testInitialAndFinalWatermark() {
	AscendingTimestampExtractor<Long> extractor = new LongExtractor();
	assertEquals(Long.MIN_VALUE, extractor.getCurrentWatermark().getTimestamp());

	extractor.extractTimestamp(Long.MIN_VALUE, -1L);

	extractor.extractTimestamp(Long.MAX_VALUE, -1L);
	assertEquals(Long.MAX_VALUE - 1, extractor.getCurrentWatermark().getTimestamp());
}
 

private void runValidTests(AscendingTimestampExtractor<Long> extractor) {
	assertEquals(13L, extractor.extractTimestamp(13L, -1L));
	assertEquals(13L, extractor.extractTimestamp(13L, 0L));
	assertEquals(14L, extractor.extractTimestamp(14L, 0L));
	assertEquals(20L, extractor.extractTimestamp(20L, 0L));
	assertEquals(20L, extractor.extractTimestamp(20L, 0L));
	assertEquals(20L, extractor.extractTimestamp(20L, 0L));
	assertEquals(500L, extractor.extractTimestamp(500L, 0L));

	assertEquals(Long.MAX_VALUE - 1, extractor.extractTimestamp(Long.MAX_VALUE - 1, 99999L));
}
 

private void runInvalidTest(AscendingTimestampExtractor<Long> extractor) {
	assertEquals(1000L, extractor.extractTimestamp(1000L, 100));
	assertEquals(1000L, extractor.extractTimestamp(1000L, 100));

	// violation
	assertEquals(999L, extractor.extractTimestamp(999L, 100));
}
 

public static void main(String[] args) throws Exception {

        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
        // 构建输入数据
        List<Tuple3<String, Long, Long>> data = new ArrayList<>();
        Tuple3<String, Long, Long> a1 = new Tuple3<>("first event", 1L, 1111L);
        Tuple3<String, Long, Long> a2 = new Tuple3<>("second event", 1L, 1112L);
        Tuple3<String, Long, Long> a3 = new Tuple3<>("third event", 1L, 20121L);
        Tuple3<String, Long, Long> b1 = new Tuple3<>("first event", 2L, 1111L);
        Tuple3<String, Long, Long> b2 = new Tuple3<>("second event", 2L, 1112L);
        Tuple3<String, Long, Long> b3 = new Tuple3<>("third event", 2L, 30111L);
        data.add(a1);
        data.add(a2);
        data.add(a3);
        data.add(b1);
        data.add(b2);
        data.add(b3);
        DataStreamSource<Tuple3<String, Long, Long>> input = env.fromCollection(data);


        input.assignTimestampsAndWatermarks(new AscendingTimestampExtractor<Tuple3<String, Long, Long>>() {
            @Override
            public long extractAscendingTimestamp(Tuple3<String, Long, Long> element) {
                return element.f2;
            }
        }).keyBy(x -> x.f1).timeWindow(Time.seconds(1), Time.seconds(1)).process(new MyProcessWindowFunction()).print();

        // 输出结果:
        // 3> window: TimeWindow{start=1000, end=2000}word count: 4
        // 4> window: TimeWindow{start=1000, end=2000}word count: 4
        // 3> window: TimeWindow{start=20000, end=21000}word count: 2
        // 4> window: TimeWindow{start=30000, end=31000}word count: 2


        env.execute();
    }
 

@Test
public void testUpsert() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.getConfig().enableObjectReuse();
	env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
	StreamTableEnvironment tEnv = StreamTableEnvironment.create(env);

	Table t = tEnv.fromDataStream(get3TupleDataStream(env).assignTimestampsAndWatermarks(
			new AscendingTimestampExtractor<Tuple3<Integer, Long, String>>() {
				@Override
				public long extractAscendingTimestamp(Tuple3<Integer, Long, String> element) {
					return element.f0;
				}}), "id, num, text");

	tEnv.registerTable("T", t);

	String[] fields = {"cnt", "lencnt", "cTag"};
	tEnv.registerTableSink("upsertSink", JDBCUpsertTableSink.builder()
			.setOptions(JDBCOptions.builder()
					.setDBUrl(DB_URL)
					.setTableName(OUTPUT_TABLE1)
					.build())
			.setTableSchema(TableSchema.builder().fields(
					fields, new DataType[] {BIGINT(), BIGINT(), INT()}).build())
			.build());

	tEnv.sqlUpdate("INSERT INTO upsertSink SELECT cnt, COUNT(len) AS lencnt, cTag FROM" +
			" (SELECT len, COUNT(id) as cnt, cTag FROM" +
			" (SELECT id, CHAR_LENGTH(text) AS len, (CASE WHEN id > 0 THEN 1 ELSE 0 END) cTag FROM T)" +
			" GROUP BY len, cTag)" +
			" GROUP BY cnt, cTag");
	env.execute();
	check(new Row[] {
			Row.of(1, 5, 1),
			Row.of(7, 1, 1),
			Row.of(9, 1, 1)
	}, DB_URL, OUTPUT_TABLE1, fields);
}
 

@Test
public void testWithFailingHandler() {
	AscendingTimestampExtractor<Long> extractor = new LongExtractor()
			.withViolationHandler(new AscendingTimestampExtractor.FailingHandler());

	runValidTests(extractor);
	try {
		runInvalidTest(extractor);
		fail("should fail with an exception");
	} catch (Exception ignored) {}
}
 

@Test
public void testWithLoggingHandler() {
	AscendingTimestampExtractor<Long> extractor = new LongExtractor()
			.withViolationHandler(new AscendingTimestampExtractor.LoggingHandler());

	runValidTests(extractor);
	runInvalidTest(extractor);
}
 

@Test
public void testUpsert() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.getConfig().enableObjectReuse();
	env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
	EnvironmentSettings envSettings = EnvironmentSettings.newInstance()
		.useBlinkPlanner()
		.inStreamingMode()
		.build();
	StreamTableEnvironment tEnv = StreamTableEnvironment.create(env, envSettings);

	Table t = tEnv.fromDataStream(get4TupleDataStream(env).assignTimestampsAndWatermarks(
		new AscendingTimestampExtractor<Tuple4<Integer, Long, String, Timestamp>>() {
			@Override
			public long extractAscendingTimestamp(Tuple4<Integer, Long, String, Timestamp> element) {
				return element.f0;
			}
		}), $("id"), $("num"), $("text"), $("ts"));

	tEnv.createTemporaryView("T", t);
	tEnv.executeSql(
		"CREATE TABLE upsertSink (" +
			"  cnt BIGINT," +
			"  lencnt BIGINT," +
			"  cTag INT," +
			"  ts TIMESTAMP(3)," +
			"  PRIMARY KEY (cnt, cTag) NOT ENFORCED" +
			") WITH (" +
			"  'connector'='jdbc'," +
			"  'url'='" + DB_URL + "'," +
			"  'table-name'='" + OUTPUT_TABLE1 + "'" +
			")");

	TableResult tableResult = tEnv.executeSql("INSERT INTO upsertSink \n" +
		"SELECT cnt, COUNT(len) AS lencnt, cTag, MAX(ts) AS ts\n" +
		"FROM (\n" +
		"  SELECT len, COUNT(id) as cnt, cTag, MAX(ts) AS ts\n" +
		"  FROM (SELECT id, CHAR_LENGTH(text) AS len, (CASE WHEN id > 0 THEN 1 ELSE 0 END) cTag, ts FROM T)\n" +
		"  GROUP BY len, cTag\n" +
		")\n" +
		"GROUP BY cnt, cTag");
	// wait to finish
	tableResult.getJobClient().get().getJobExecutionResult(Thread.currentThread().getContextClassLoader()).get();
	check(new Row[] {
		Row.of(1, 5, 1, Timestamp.valueOf("1970-01-01 00:00:00.006")),
		Row.of(7, 1, 1, Timestamp.valueOf("1970-01-01 00:00:00.021")),
		Row.of(9, 1, 1, Timestamp.valueOf("1970-01-01 00:00:00.015"))
	}, DB_URL, OUTPUT_TABLE1, new String[]{"cnt", "lencnt", "cTag", "ts"});
}
 

@Test
public void testUpsert() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.getConfig().enableObjectReuse();
	env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
	StreamTableEnvironment tEnv = StreamTableEnvironment.create(env);

	Table t = tEnv.fromDataStream(get4TupleDataStream(env).assignTimestampsAndWatermarks(
		new AscendingTimestampExtractor<Tuple4<Integer, Long, String, Timestamp>>() {
			@Override
			public long extractAscendingTimestamp(Tuple4<Integer, Long, String, Timestamp> element) {
				return element.f0;
			}
		}), $("id"), $("num"), $("text"), $("ts"));

	tEnv.createTemporaryView("T", t);
	tEnv.executeSql(
		"CREATE TABLE upsertSink (" +
			"  cnt BIGINT," +
			"  lencnt BIGINT," +
			"  cTag INT," +
			"  ts TIMESTAMP(3)" +
			") WITH (" +
			"  'connector.type'='jdbc'," +
			"  'connector.url'='" + DB_URL + "'," +
			"  'connector.table'='" + OUTPUT_TABLE1 + "'" +
			")");

	TableResult tableResult = tEnv.executeSql("INSERT INTO upsertSink \n" +
		"SELECT cnt, COUNT(len) AS lencnt, cTag, MAX(ts) AS ts\n" +
		"FROM (\n" +
		"  SELECT len, COUNT(id) as cnt, cTag, MAX(ts) AS ts\n" +
		"  FROM (SELECT id, CHAR_LENGTH(text) AS len, (CASE WHEN id > 0 THEN 1 ELSE 0 END) cTag, ts FROM T)\n" +
		"  GROUP BY len, cTag\n" +
		")\n" +
		"GROUP BY cnt, cTag");
	// wait to finish
	tableResult.getJobClient().get().getJobExecutionResult(Thread.currentThread().getContextClassLoader()).get();
	check(new Row[] {
			Row.of(1, 5, 1, Timestamp.valueOf("1970-01-01 00:00:00.006")),
			Row.of(7, 1, 1, Timestamp.valueOf("1970-01-01 00:00:00.021")),
			Row.of(9, 1, 1, Timestamp.valueOf("1970-01-01 00:00:00.015"))
	}, DB_URL, OUTPUT_TABLE1, new String[]{"cnt", "lencnt", "cTag", "ts"});
}
 

/**
 * This tests whether timestamps are properly extracted in the timestamp
 * extractor and whether watermarks are also correctly forwarded from this with the auto watermark
 * interval.
 */
@Test
public void testTimestampExtractorWithAutoInterval() throws Exception {
	final int numElements = 10;

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
	env.getConfig().setAutoWatermarkInterval(10);
	env.setParallelism(1);

	DataStream<Integer> source1 = env.addSource(new SourceFunction<Integer>() {
		@Override
		public void run(SourceContext<Integer> ctx) throws Exception {
			int index = 1;
			while (index <= numElements) {
				ctx.collect(index);
				latch.await();
				index++;
			}
		}

		@Override
		public void cancel() {}
	});

	DataStream<Integer> extractOp = source1.assignTimestampsAndWatermarks(
			new AscendingTimestampExtractor<Integer>() {
				@Override
				public long extractAscendingTimestamp(Integer element) {
					return element;
				}
			});

	extractOp
			.transform("Watermark Check", BasicTypeInfo.INT_TYPE_INFO, new CustomOperator(true))
			.transform("Timestamp Check",
					BasicTypeInfo.INT_TYPE_INFO,
					new TimestampCheckingOperator());

	// verify that extractor picks up source parallelism
	Assert.assertEquals(extractOp.getTransformation().getParallelism(), source1.getTransformation().getParallelism());

	env.execute();

	// verify that we get NUM_ELEMENTS watermarks
	for (int j = 0; j < numElements; j++) {
		if (!CustomOperator.finalWatermarks[0].get(j).equals(new Watermark(j))) {
			long wm = CustomOperator.finalWatermarks[0].get(j).getTimestamp();
			Assert.fail("Wrong watermark. Expected: " + j + " Found: " + wm + " All: " + CustomOperator.finalWatermarks[0]);
		}
	}

	// the input is finite, so it should have a MAX Watermark
	assertEquals(Watermark.MAX_WATERMARK,
			CustomOperator.finalWatermarks[0].get(CustomOperator.finalWatermarks[0].size() - 1));
}