Java Code Examples for org.apache.flink.streaming.util.KeyedTwoInputStreamOperatorTestHarness#close()
The following examples show how to use
org.apache.flink.streaming.util.KeyedTwoInputStreamOperatorTestHarness#close() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: RowTimeBoundedStreamJoinTest.java From flink with Apache License 2.0 | 4 votes |
|
/** a.rowtime >= b.rowtime - 10 and a.rowtime <= b.rowtime + 20. **/
@Test
public void testRowTimeInnerJoinWithCommonBounds() throws Exception {
RowTimeBoundedStreamJoin joinProcessFunc = new RowTimeBoundedStreamJoin(
FlinkJoinType.INNER, -10, 20, 0, rowType, rowType, generatedFunction, 0, 0);
KeyedTwoInputStreamOperatorTestHarness<BaseRow, BaseRow, BaseRow, BaseRow> testHarness = createTestHarness(
joinProcessFunc);
testHarness.open();
testHarness.processWatermark1(new Watermark(1));
testHarness.processWatermark2(new Watermark(1));
// Test late data.
testHarness.processElement1(record(1L, "k1"));
// Though (1L, "k1") is actually late, it will also be cached.
assertEquals(1, testHarness.numEventTimeTimers());
testHarness.processElement1(record(2L, "k1"));
testHarness.processElement2(record(2L, "k1"));
assertEquals(2, testHarness.numEventTimeTimers());
assertEquals(4, testHarness.numKeyedStateEntries());
testHarness.processElement1(record(5L, "k1"));
testHarness.processElement2(record(15L, "k1"));
testHarness.processWatermark1(new Watermark(20));
testHarness.processWatermark2(new Watermark(20));
assertEquals(4, testHarness.numKeyedStateEntries());
testHarness.processElement1(record(35L, "k1"));
// The right rows with timestamp = 2 and 5 will be removed here.
// The left rows with timestamp = 2 and 15 will be removed here.
testHarness.processWatermark1(new Watermark(38));
testHarness.processWatermark2(new Watermark(38));
testHarness.processElement1(record(40L, "k2"));
testHarness.processElement2(record(39L, "k2"));
assertEquals(6, testHarness.numKeyedStateEntries());
// The right row with timestamp = 35 will be removed here.
testHarness.processWatermark1(new Watermark(61));
testHarness.processWatermark2(new Watermark(61));
assertEquals(4, testHarness.numKeyedStateEntries());
List<Object> expectedOutput = new ArrayList<>();
expectedOutput.add(new Watermark(-19));
// This result is produced by the late row (1, "k1").
expectedOutput.add(record(1L, "k1", 2L, "k1"));
expectedOutput.add(record(2L, "k1", 2L, "k1"));
expectedOutput.add(record(5L, "k1", 2L, "k1"));
expectedOutput.add(record(5L, "k1", 15L, "k1"));
expectedOutput.add(new Watermark(0));
expectedOutput.add(record(35L, "k1", 15L, "k1"));
expectedOutput.add(new Watermark(18));
expectedOutput.add(record(40L, "k2", 39L, "k2"));
expectedOutput.add(new Watermark(41));
assertor.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
testHarness.close();
}
Example 2
| Source File: RowTimeBoundedStreamJoinTest.java From flink with Apache License 2.0 | 4 votes |
|
/** a.rowtime >= b.rowtime - 10 and a.rowtime <= b.rowtime - 7. **/
@Test
public void testRowTimeInnerJoinWithNegativeBounds() throws Exception {
RowTimeBoundedStreamJoin joinProcessFunc = new RowTimeBoundedStreamJoin(
FlinkJoinType.INNER, -10, -7, 0, rowType, rowType, generatedFunction, 0, 0);
KeyedTwoInputStreamOperatorTestHarness<BaseRow, BaseRow, BaseRow, BaseRow> testHarness = createTestHarness(
joinProcessFunc);
testHarness.open();
testHarness.processWatermark1(new Watermark(1));
testHarness.processWatermark2(new Watermark(1));
// This row will not be cached.
testHarness.processElement2(record(2L, "k1"));
assertEquals(0, testHarness.numKeyedStateEntries());
testHarness.processWatermark1(new Watermark(2));
testHarness.processWatermark2(new Watermark(2));
testHarness.processElement1(record(3L, "k1"));
testHarness.processElement2(record(3L, "k1"));
// Test for -10 boundary (13 - 10 = 3).
// This row from the right stream will be cached.
// The clean time for the left stream is 13 - 7 + 1 - 1 = 8
testHarness.processElement2(record(13L, "k1"));
// Test for -7 boundary (13 - 7 = 6).
testHarness.processElement1(record(6L, "k1"));
assertEquals(4, testHarness.numKeyedStateEntries());
// Trigger the left timer with timestamp 8.
// The row with timestamp = 13 will be removed here (13 < 10 + 7).
testHarness.processWatermark1(new Watermark(10));
testHarness.processWatermark2(new Watermark(10));
assertEquals(2, testHarness.numKeyedStateEntries());
// Clear the states.
testHarness.processWatermark1(new Watermark(18));
testHarness.processWatermark2(new Watermark(18));
assertEquals(0, testHarness.numKeyedStateEntries());
List<Object> expectedOutput = new ArrayList<>();
expectedOutput.add(new Watermark(-9));
expectedOutput.add(new Watermark(-8));
expectedOutput.add(record(3L, "k1", 13L, "k1"));
expectedOutput.add(record(6L, "k1", 13L, "k1"));
expectedOutput.add(new Watermark(0));
expectedOutput.add(new Watermark(8));
assertor.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
testHarness.close();
}
Example 3
| Source File: RowTimeBoundedStreamJoinTest.java From flink with Apache License 2.0 | 4 votes |
|
@Test
public void testRowTimeLeftOuterJoin() throws Exception {
RowTimeBoundedStreamJoin joinProcessFunc = new RowTimeBoundedStreamJoin(
FlinkJoinType.LEFT, -5, 9, 0, rowType, rowType, generatedFunction, 0, 0);
KeyedTwoInputStreamOperatorTestHarness<BaseRow, BaseRow, BaseRow, BaseRow> testHarness = createTestHarness(
joinProcessFunc);
testHarness.open();
testHarness.processElement1(record(1L, "k1"));
testHarness.processElement2(record(1L, "k2"));
assertEquals(2, testHarness.numEventTimeTimers());
assertEquals(4, testHarness.numKeyedStateEntries());
// The left row with timestamp = 1 will be padded and removed (14=1+5+1+((5+9)/2)).
testHarness.processWatermark1(new Watermark(14));
testHarness.processWatermark2(new Watermark(14));
assertEquals(1, testHarness.numEventTimeTimers());
assertEquals(2, testHarness.numKeyedStateEntries());
// The right row with timestamp = 1 will be removed (18=1+9+1+((5+9)/2)).
testHarness.processWatermark1(new Watermark(18));
testHarness.processWatermark2(new Watermark(18));
assertEquals(0, testHarness.numEventTimeTimers());
assertEquals(0, testHarness.numKeyedStateEntries());
testHarness.processElement1(record(2L, "k1"));
testHarness.processElement2(record(2L, "k2"));
// The late rows with timestamp = 2 will not be cached, but a null padding result for the left
// row will be emitted.
assertEquals(0, testHarness.numKeyedStateEntries());
assertEquals(0, testHarness.numEventTimeTimers());
// Make sure the common (inner) join can be performed.
testHarness.processElement1(record(19L, "k1"));
testHarness.processElement1(record(20L, "k1"));
testHarness.processElement2(record(26L, "k1"));
testHarness.processElement2(record(25L, "k1"));
testHarness.processElement1(record(21L, "k1"));
testHarness.processElement2(record(39L, "k2"));
testHarness.processElement2(record(40L, "k2"));
testHarness.processElement1(record(50L, "k2"));
testHarness.processElement1(record(49L, "k2"));
testHarness.processElement2(record(41L, "k2"));
testHarness.processWatermark1(new Watermark(100));
testHarness.processWatermark2(new Watermark(100));
List<Object> expectedOutput = new ArrayList<>();
// The timestamp 14 is set with the triggered timer.
expectedOutput.add(record(1L, "k1", null, null));
expectedOutput.add(new Watermark(5));
expectedOutput.add(new Watermark(9));
expectedOutput.add(record(2L, "k1", null, null));
expectedOutput.add(record(20L, "k1", 25L, "k1"));
expectedOutput.add(record(21L, "k1", 25L, "k1"));
expectedOutput.add(record(21L, "k1", 26L, "k1"));
expectedOutput.add(record(49L, "k2", 40L, "k2"));
expectedOutput.add(record(49L, "k2", 41L, "k2"));
expectedOutput.add(record(50L, "k2", 41L, "k2"));
// The timestamp 32 is set with the triggered timer.
expectedOutput.add(record(19L, "k1", null, null));
expectedOutput.add(new Watermark(91));
assertor.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
testHarness.close();
}
Example 4
| Source File: RowTimeBoundedStreamJoinTest.java From flink with Apache License 2.0 | 4 votes |
|
@Test
public void testRowTimeRightOuterJoin() throws Exception {
RowTimeBoundedStreamJoin joinProcessFunc = new RowTimeBoundedStreamJoin(
FlinkJoinType.RIGHT, -5, 9, 0, rowType, rowType, generatedFunction, 0, 0);
KeyedTwoInputStreamOperatorTestHarness<BaseRow, BaseRow, BaseRow, BaseRow> testHarness = createTestHarness(
joinProcessFunc);
testHarness.open();
testHarness.processElement1(record(1L, "k1"));
testHarness.processElement2(record(1L, "k2"));
assertEquals(2, testHarness.numEventTimeTimers());
assertEquals(4, testHarness.numKeyedStateEntries());
// The left row with timestamp = 1 will be removed (14=1+5+1+((5+9)/2)).
testHarness.processWatermark1(new Watermark(14));
testHarness.processWatermark2(new Watermark(14));
assertEquals(1, testHarness.numEventTimeTimers());
assertEquals(2, testHarness.numKeyedStateEntries());
// The right row with timestamp = 1 will be padded and removed (18=1+9+1+((5+9)/2)).
testHarness.processWatermark1(new Watermark(18));
testHarness.processWatermark2(new Watermark(18));
assertEquals(0, testHarness.numEventTimeTimers());
assertEquals(0, testHarness.numKeyedStateEntries());
testHarness.processElement1(record(2L, "k1"));
testHarness.processElement2(record(2L, "k2"));
// The late rows with timestamp = 2 will not be cached, but a null padding result for the right
// row will be emitted.
assertEquals(0, testHarness.numKeyedStateEntries());
assertEquals(0, testHarness.numEventTimeTimers());
// Make sure the common (inner) join can be performed.
testHarness.processElement1(record(19L, "k1"));
testHarness.processElement1(record(20L, "k1"));
testHarness.processElement2(record(26L, "k1"));
testHarness.processElement2(record(25L, "k1"));
testHarness.processElement1(record(21L, "k1"));
testHarness.processElement2(record(39L, "k2"));
testHarness.processElement2(record(40L, "k2"));
testHarness.processElement1(record(50L, "k2"));
testHarness.processElement1(record(49L, "k2"));
testHarness.processElement2(record(41L, "k2"));
testHarness.processWatermark1(new Watermark(100));
testHarness.processWatermark2(new Watermark(100));
List<Object> expectedOutput = new ArrayList<>();
expectedOutput.add(new Watermark(5));
// The timestamp 18 is set with the triggered timer.
expectedOutput.add(record(null, null, 1L, "k2"));
expectedOutput.add(new Watermark(9));
expectedOutput.add(record(null, null, 2L, "k2"));
expectedOutput.add(record(20L, "k1", 25L, "k1"));
expectedOutput.add(record(21L, "k1", 25L, "k1"));
expectedOutput.add(record(21L, "k1", 26L, "k1"));
expectedOutput.add(record(49L, "k2", 40L, "k2"));
expectedOutput.add(record(49L, "k2", 41L, "k2"));
expectedOutput.add(record(50L, "k2", 41L, "k2"));
// The timestamp 56 is set with the triggered timer.
expectedOutput.add(record(null, null, 39L, "k2"));
expectedOutput.add(new Watermark(91));
assertor.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
testHarness.close();
}
Example 5
| Source File: RowTimeBoundedStreamJoinTest.java From flink with Apache License 2.0 | 4 votes |
|
/** a.rowtime >= b.rowtime - 5 and a.rowtime <= b.rowtime + 9. **/
@Test
public void testRowTimeFullOuterJoin() throws Exception {
RowTimeBoundedStreamJoin joinProcessFunc = new RowTimeBoundedStreamJoin(
FlinkJoinType.FULL, -5, 9, 0, rowType, rowType, generatedFunction, 0, 0);
KeyedTwoInputStreamOperatorTestHarness<BaseRow, BaseRow, BaseRow, BaseRow> testHarness = createTestHarness(
joinProcessFunc);
testHarness.open();
testHarness.processElement1(record(1L, "k1"));
testHarness.processElement2(record(1L, "k2"));
assertEquals(2, testHarness.numEventTimeTimers());
assertEquals(4, testHarness.numKeyedStateEntries());
// The left row with timestamp = 1 will be padded and removed (14=1+5+1+((5+9)/2)).
testHarness.processWatermark1(new Watermark(14));
testHarness.processWatermark2(new Watermark(14));
assertEquals(1, testHarness.numEventTimeTimers());
assertEquals(2, testHarness.numKeyedStateEntries());
// The right row with timestamp = 1 will be padded and removed (18=1+9+1+((5+9)/2)).
testHarness.processWatermark1(new Watermark(18));
testHarness.processWatermark2(new Watermark(18));
assertEquals(0, testHarness.numEventTimeTimers());
assertEquals(0, testHarness.numKeyedStateEntries());
testHarness.processElement1(record(2L, "k1"));
testHarness.processElement2(record(2L, "k2"));
// The late rows with timestamp = 2 will not be cached, but a null padding result for the right
// row will be emitted.
assertEquals(0, testHarness.numKeyedStateEntries());
assertEquals(0, testHarness.numEventTimeTimers());
// Make sure the common (inner) join can be performed.
testHarness.processElement1(record(19L, "k1"));
testHarness.processElement1(record(20L, "k1"));
testHarness.processElement2(record(26L, "k1"));
testHarness.processElement2(record(25L, "k1"));
testHarness.processElement1(record(21L, "k1"));
testHarness.processElement2(record(39L, "k2"));
testHarness.processElement2(record(40L, "k2"));
testHarness.processElement1(record(50L, "k2"));
testHarness.processElement1(record(49L, "k2"));
testHarness.processElement2(record(41L, "k2"));
testHarness.processWatermark1(new Watermark(100));
testHarness.processWatermark2(new Watermark(100));
List<Object> expectedOutput = new ArrayList<>();
// The timestamp 14 is set with the triggered timer.
expectedOutput.add(record(1L, "k1", null, null));
expectedOutput.add(new Watermark(5));
// The timestamp 18 is set with the triggered timer.
expectedOutput.add(record(null, null, 1L, "k2"));
expectedOutput.add(new Watermark(9));
expectedOutput.add(record(2L, "k1", null, null));
expectedOutput.add(record(null, null, 2L, "k2"));
expectedOutput.add(record(20L, "k1", 25L, "k1"));
expectedOutput.add(record(21L, "k1", 25L, "k1"));
expectedOutput.add(record(21L, "k1", 26L, "k1"));
expectedOutput.add(record(49L, "k2", 40L, "k2"));
expectedOutput.add(record(49L, "k2", 41L, "k2"));
expectedOutput.add(record(50L, "k2", 41L, "k2"));
// The timestamp 32 is set with the triggered timer.
expectedOutput.add(record(19L, "k1", null, null));
// The timestamp 56 is set with the triggered timer.
expectedOutput.add(record(null, null, 39L, "k2"));
expectedOutput.add(new Watermark(91));
assertor.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
testHarness.close();
}
Example 6
| Source File: ProcTimeBoundedStreamJoinTest.java From flink with Apache License 2.0 | 4 votes |
|
/** a.proctime >= b.proctime - 10 and a.proctime <= b.proctime + 20. **/
@Test
public void testProcTimeInnerJoinWithCommonBounds() throws Exception {
ProcTimeBoundedStreamJoin joinProcessFunc = new ProcTimeBoundedStreamJoin(
FlinkJoinType.INNER, -10, 20, rowType, rowType, generatedFunction);
KeyedTwoInputStreamOperatorTestHarness<BaseRow, BaseRow, BaseRow, BaseRow> testHarness = createTestHarness(
joinProcessFunc);
testHarness.open();
testHarness.setProcessingTime(1);
testHarness.processElement1(record(1L, "1a1"));
assertEquals(1, testHarness.numProcessingTimeTimers());
testHarness.setProcessingTime(2);
testHarness.processElement1(record(2L, "2a2"));
// timers for key = 1 and key = 2
assertEquals(2, testHarness.numProcessingTimeTimers());
testHarness.setProcessingTime(3);
testHarness.processElement1(record(1L, "1a3"));
assertEquals(4, testHarness.numKeyedStateEntries());
// The number of timers won't increase.
assertEquals(2, testHarness.numProcessingTimeTimers());
testHarness.processElement2(record(1L, "1b3"));
testHarness.setProcessingTime(4);
testHarness.processElement2(record(2L, "2b4"));
// The number of states should be doubled.
assertEquals(8, testHarness.numKeyedStateEntries());
assertEquals(4, testHarness.numProcessingTimeTimers());
// Test for -10 boundary (13 - 10 = 3).
// The left row (key = 1) with timestamp = 1 will be eagerly removed here.
testHarness.setProcessingTime(13);
testHarness.processElement2(record(1L, "1b13"));
// Test for +20 boundary (13 + 20 = 33).
testHarness.setProcessingTime(33);
assertEquals(4, testHarness.numKeyedStateEntries());
assertEquals(2, testHarness.numProcessingTimeTimers());
testHarness.processElement1(record(1L, "1a33"));
testHarness.processElement1(record(2L, "2a33"));
// The left row (key = 2) with timestamp = 2 will be eagerly removed here.
testHarness.processElement2(record(2L, "2b33"));
List<Object> expectedOutput = new ArrayList<>();
expectedOutput.add(record(1L, "1a1", 1L, "1b3"));
expectedOutput.add(record(1L, "1a3", 1L, "1b3"));
expectedOutput.add(record(2L, "2a2", 2L, "2b4"));
expectedOutput.add(record(1L, "1a3", 1L, "1b13"));
expectedOutput.add(record(1L, "1a33", 1L, "1b13"));
expectedOutput.add(record(2L, "2a33", 2L, "2b33"));
assertor.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
testHarness.close();
}
Example 7
| Source File: ProcTimeBoundedStreamJoinTest.java From flink with Apache License 2.0 | 4 votes |
|
/** a.proctime >= b.proctime - 10 and a.proctime <= b.proctime - 5. **/
@Test
public void testProcTimeInnerJoinWithNegativeBounds() throws Exception {
ProcTimeBoundedStreamJoin joinProcessFunc = new ProcTimeBoundedStreamJoin(
FlinkJoinType.INNER, -10, -5, rowType, rowType, generatedFunction);
KeyedTwoInputStreamOperatorTestHarness<BaseRow, BaseRow, BaseRow, BaseRow> testHarness = createTestHarness(
joinProcessFunc);
testHarness.open();
testHarness.setProcessingTime(1);
testHarness.processElement1(record(1L, "1a1"));
testHarness.setProcessingTime(2);
testHarness.processElement1(record(2L, "2a2"));
testHarness.setProcessingTime(3);
testHarness.processElement1(record(1L, "1a3"));
assertEquals(4, testHarness.numKeyedStateEntries());
assertEquals(2, testHarness.numProcessingTimeTimers());
// All the right rows will not be cached.
testHarness.processElement2(record(1L, "1b3"));
assertEquals(4, testHarness.numKeyedStateEntries());
assertEquals(2, testHarness.numProcessingTimeTimers());
testHarness.setProcessingTime(7);
// Meets a.proctime <= b.proctime - 5.
// This row will only be joined without being cached (7 >= 7 - 5).
testHarness.processElement2(record(2L, "2b7"));
assertEquals(4, testHarness.numKeyedStateEntries());
assertEquals(2, testHarness.numProcessingTimeTimers());
testHarness.setProcessingTime(12);
// The left row (key = 1) with timestamp = 1 will be eagerly removed here.
testHarness.processElement2(record(1L, "1b12"));
// We add a delay (relativeWindowSize / 2) for cleaning up state.
// No timers will be triggered here.
testHarness.setProcessingTime(13);
assertEquals(4, testHarness.numKeyedStateEntries());
assertEquals(2, testHarness.numProcessingTimeTimers());
// Trigger the timer registered by the left row (key = 1) with timestamp = 1
// (1 + 10 + 2 + 0 + 1 = 14).
// The left row (key = 1) with timestamp = 3 will removed here.
testHarness.setProcessingTime(14);
assertEquals(2, testHarness.numKeyedStateEntries());
assertEquals(1, testHarness.numProcessingTimeTimers());
// Clean up the left row (key = 2) with timestamp = 2.
testHarness.setProcessingTime(16);
assertEquals(0, testHarness.numKeyedStateEntries());
assertEquals(0, testHarness.numProcessingTimeTimers());
List<Object> expectedOutput = new ArrayList<>();
expectedOutput.add(record(2L, "2a2", 2L, "2b7"));
expectedOutput.add(record(1L, "1a3", 1L, "1b12"));
assertor.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
testHarness.close();
}
Example 8
| Source File: UrlDBFunctionTest.java From flink-crawler with Apache License 2.0 | 4 votes |
|
private void closeTestHarnesses() throws Exception {
for (KeyedTwoInputStreamOperatorTestHarness<String, CrawlStateUrl, DomainScore, FetchUrl> harness : _testHarnesses) {
harness.close();
}
}
Example 9
| Source File: RowTimeIntervalJoinTest.java From flink with Apache License 2.0 | 4 votes |
|
/** a.rowtime >= b.rowtime - 10 and a.rowtime <= b.rowtime + 20. **/
@Test
public void testRowTimeInnerJoinWithCommonBounds() throws Exception {
RowTimeIntervalJoin joinProcessFunc = new RowTimeIntervalJoin(
FlinkJoinType.INNER, -10, 20, 0, rowType, rowType, generatedFunction, 0, 0);
KeyedTwoInputStreamOperatorTestHarness<RowData, RowData, RowData, RowData> testHarness = createTestHarness(
joinProcessFunc);
testHarness.open();
testHarness.processWatermark1(new Watermark(1));
testHarness.processWatermark2(new Watermark(1));
// Test late data.
testHarness.processElement1(insertRecord(1L, "k1"));
// Though (1L, "k1") is actually late, it will also be cached.
assertEquals(1, testHarness.numEventTimeTimers());
testHarness.processElement1(insertRecord(2L, "k1"));
testHarness.processElement2(insertRecord(2L, "k1"));
assertEquals(2, testHarness.numEventTimeTimers());
assertEquals(4, testHarness.numKeyedStateEntries());
testHarness.processElement1(insertRecord(5L, "k1"));
testHarness.processElement2(insertRecord(15L, "k1"));
testHarness.processWatermark1(new Watermark(20));
testHarness.processWatermark2(new Watermark(20));
assertEquals(4, testHarness.numKeyedStateEntries());
testHarness.processElement1(insertRecord(35L, "k1"));
// The right rows with timestamp = 2 and 5 will be removed here.
// The left rows with timestamp = 2 and 15 will be removed here.
testHarness.processWatermark1(new Watermark(38));
testHarness.processWatermark2(new Watermark(38));
testHarness.processElement1(insertRecord(40L, "k2"));
testHarness.processElement2(insertRecord(39L, "k2"));
assertEquals(6, testHarness.numKeyedStateEntries());
// The right row with timestamp = 35 will be removed here.
testHarness.processWatermark1(new Watermark(61));
testHarness.processWatermark2(new Watermark(61));
assertEquals(4, testHarness.numKeyedStateEntries());
List<Object> expectedOutput = new ArrayList<>();
expectedOutput.add(new Watermark(-19));
// This result is produced by the late row (1, "k1").
expectedOutput.add(insertRecord(1L, "k1", 2L, "k1"));
expectedOutput.add(insertRecord(2L, "k1", 2L, "k1"));
expectedOutput.add(insertRecord(5L, "k1", 2L, "k1"));
expectedOutput.add(insertRecord(5L, "k1", 15L, "k1"));
expectedOutput.add(new Watermark(0));
expectedOutput.add(insertRecord(35L, "k1", 15L, "k1"));
expectedOutput.add(new Watermark(18));
expectedOutput.add(insertRecord(40L, "k2", 39L, "k2"));
expectedOutput.add(new Watermark(41));
assertor.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
testHarness.close();
}
Example 10
| Source File: RowTimeIntervalJoinTest.java From flink with Apache License 2.0 | 4 votes |
|
/** a.rowtime >= b.rowtime - 10 and a.rowtime <= b.rowtime - 7. **/
@Test
public void testRowTimeInnerJoinWithNegativeBounds() throws Exception {
RowTimeIntervalJoin joinProcessFunc = new RowTimeIntervalJoin(
FlinkJoinType.INNER, -10, -7, 0, rowType, rowType, generatedFunction, 0, 0);
KeyedTwoInputStreamOperatorTestHarness<RowData, RowData, RowData, RowData> testHarness = createTestHarness(
joinProcessFunc);
testHarness.open();
testHarness.processWatermark1(new Watermark(1));
testHarness.processWatermark2(new Watermark(1));
// This row will not be cached.
testHarness.processElement2(insertRecord(2L, "k1"));
assertEquals(0, testHarness.numKeyedStateEntries());
testHarness.processWatermark1(new Watermark(2));
testHarness.processWatermark2(new Watermark(2));
testHarness.processElement1(insertRecord(3L, "k1"));
testHarness.processElement2(insertRecord(3L, "k1"));
// Test for -10 boundary (13 - 10 = 3).
// This row from the right stream will be cached.
// The clean time for the left stream is 13 - 7 + 1 - 1 = 8
testHarness.processElement2(insertRecord(13L, "k1"));
// Test for -7 boundary (13 - 7 = 6).
testHarness.processElement1(insertRecord(6L, "k1"));
assertEquals(4, testHarness.numKeyedStateEntries());
// Trigger the left timer with timestamp 8.
// The row with timestamp = 13 will be removed here (13 < 10 + 7).
testHarness.processWatermark1(new Watermark(10));
testHarness.processWatermark2(new Watermark(10));
assertEquals(2, testHarness.numKeyedStateEntries());
// Clear the states.
testHarness.processWatermark1(new Watermark(18));
testHarness.processWatermark2(new Watermark(18));
assertEquals(0, testHarness.numKeyedStateEntries());
List<Object> expectedOutput = new ArrayList<>();
expectedOutput.add(new Watermark(-9));
expectedOutput.add(new Watermark(-8));
expectedOutput.add(insertRecord(3L, "k1", 13L, "k1"));
expectedOutput.add(insertRecord(6L, "k1", 13L, "k1"));
expectedOutput.add(new Watermark(0));
expectedOutput.add(new Watermark(8));
assertor.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
testHarness.close();
}
Example 11
| Source File: RowTimeIntervalJoinTest.java From flink with Apache License 2.0 | 4 votes |
|
@Test
public void testRowTimeLeftOuterJoin() throws Exception {
RowTimeIntervalJoin joinProcessFunc = new RowTimeIntervalJoin(
FlinkJoinType.LEFT, -5, 9, 0, rowType, rowType, generatedFunction, 0, 0);
KeyedTwoInputStreamOperatorTestHarness<RowData, RowData, RowData, RowData> testHarness = createTestHarness(
joinProcessFunc);
testHarness.open();
testHarness.processElement1(insertRecord(1L, "k1"));
testHarness.processElement2(insertRecord(1L, "k2"));
assertEquals(2, testHarness.numEventTimeTimers());
assertEquals(4, testHarness.numKeyedStateEntries());
// The left row with timestamp = 1 will be padded and removed (14=1+5+1+((5+9)/2)).
testHarness.processWatermark1(new Watermark(14));
testHarness.processWatermark2(new Watermark(14));
assertEquals(1, testHarness.numEventTimeTimers());
assertEquals(2, testHarness.numKeyedStateEntries());
// The right row with timestamp = 1 will be removed (18=1+9+1+((5+9)/2)).
testHarness.processWatermark1(new Watermark(18));
testHarness.processWatermark2(new Watermark(18));
assertEquals(0, testHarness.numEventTimeTimers());
assertEquals(0, testHarness.numKeyedStateEntries());
testHarness.processElement1(insertRecord(2L, "k1"));
testHarness.processElement2(insertRecord(2L, "k2"));
// The late rows with timestamp = 2 will not be cached, but a null padding result for the left
// row will be emitted.
assertEquals(0, testHarness.numKeyedStateEntries());
assertEquals(0, testHarness.numEventTimeTimers());
// Make sure the common (inner) join can be performed.
testHarness.processElement1(insertRecord(19L, "k1"));
testHarness.processElement1(insertRecord(20L, "k1"));
testHarness.processElement2(insertRecord(26L, "k1"));
testHarness.processElement2(insertRecord(25L, "k1"));
testHarness.processElement1(insertRecord(21L, "k1"));
testHarness.processElement2(insertRecord(39L, "k2"));
testHarness.processElement2(insertRecord(40L, "k2"));
testHarness.processElement1(insertRecord(50L, "k2"));
testHarness.processElement1(insertRecord(49L, "k2"));
testHarness.processElement2(insertRecord(41L, "k2"));
testHarness.processWatermark1(new Watermark(100));
testHarness.processWatermark2(new Watermark(100));
List<Object> expectedOutput = new ArrayList<>();
// The timestamp 14 is set with the triggered timer.
expectedOutput.add(insertRecord(1L, "k1", null, null));
expectedOutput.add(new Watermark(5));
expectedOutput.add(new Watermark(9));
expectedOutput.add(insertRecord(2L, "k1", null, null));
expectedOutput.add(insertRecord(20L, "k1", 25L, "k1"));
expectedOutput.add(insertRecord(21L, "k1", 25L, "k1"));
expectedOutput.add(insertRecord(21L, "k1", 26L, "k1"));
expectedOutput.add(insertRecord(49L, "k2", 40L, "k2"));
expectedOutput.add(insertRecord(49L, "k2", 41L, "k2"));
expectedOutput.add(insertRecord(50L, "k2", 41L, "k2"));
// The timestamp 32 is set with the triggered timer.
expectedOutput.add(insertRecord(19L, "k1", null, null));
expectedOutput.add(new Watermark(91));
assertor.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
testHarness.close();
}
Example 12
| Source File: RowTimeIntervalJoinTest.java From flink with Apache License 2.0 | 4 votes |
|
@Test
public void testRowTimeRightOuterJoin() throws Exception {
RowTimeIntervalJoin joinProcessFunc = new RowTimeIntervalJoin(
FlinkJoinType.RIGHT, -5, 9, 0, rowType, rowType, generatedFunction, 0, 0);
KeyedTwoInputStreamOperatorTestHarness<RowData, RowData, RowData, RowData> testHarness = createTestHarness(
joinProcessFunc);
testHarness.open();
testHarness.processElement1(insertRecord(1L, "k1"));
testHarness.processElement2(insertRecord(1L, "k2"));
assertEquals(2, testHarness.numEventTimeTimers());
assertEquals(4, testHarness.numKeyedStateEntries());
// The left row with timestamp = 1 will be removed (14=1+5+1+((5+9)/2)).
testHarness.processWatermark1(new Watermark(14));
testHarness.processWatermark2(new Watermark(14));
assertEquals(1, testHarness.numEventTimeTimers());
assertEquals(2, testHarness.numKeyedStateEntries());
// The right row with timestamp = 1 will be padded and removed (18=1+9+1+((5+9)/2)).
testHarness.processWatermark1(new Watermark(18));
testHarness.processWatermark2(new Watermark(18));
assertEquals(0, testHarness.numEventTimeTimers());
assertEquals(0, testHarness.numKeyedStateEntries());
testHarness.processElement1(insertRecord(2L, "k1"));
testHarness.processElement2(insertRecord(2L, "k2"));
// The late rows with timestamp = 2 will not be cached, but a null padding result for the right
// row will be emitted.
assertEquals(0, testHarness.numKeyedStateEntries());
assertEquals(0, testHarness.numEventTimeTimers());
// Make sure the common (inner) join can be performed.
testHarness.processElement1(insertRecord(19L, "k1"));
testHarness.processElement1(insertRecord(20L, "k1"));
testHarness.processElement2(insertRecord(26L, "k1"));
testHarness.processElement2(insertRecord(25L, "k1"));
testHarness.processElement1(insertRecord(21L, "k1"));
testHarness.processElement2(insertRecord(39L, "k2"));
testHarness.processElement2(insertRecord(40L, "k2"));
testHarness.processElement1(insertRecord(50L, "k2"));
testHarness.processElement1(insertRecord(49L, "k2"));
testHarness.processElement2(insertRecord(41L, "k2"));
testHarness.processWatermark1(new Watermark(100));
testHarness.processWatermark2(new Watermark(100));
List<Object> expectedOutput = new ArrayList<>();
expectedOutput.add(new Watermark(5));
// The timestamp 18 is set with the triggered timer.
expectedOutput.add(insertRecord(null, null, 1L, "k2"));
expectedOutput.add(new Watermark(9));
expectedOutput.add(insertRecord(null, null, 2L, "k2"));
expectedOutput.add(insertRecord(20L, "k1", 25L, "k1"));
expectedOutput.add(insertRecord(21L, "k1", 25L, "k1"));
expectedOutput.add(insertRecord(21L, "k1", 26L, "k1"));
expectedOutput.add(insertRecord(49L, "k2", 40L, "k2"));
expectedOutput.add(insertRecord(49L, "k2", 41L, "k2"));
expectedOutput.add(insertRecord(50L, "k2", 41L, "k2"));
// The timestamp 56 is set with the triggered timer.
expectedOutput.add(insertRecord(null, null, 39L, "k2"));
expectedOutput.add(new Watermark(91));
assertor.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
testHarness.close();
}
Example 13
| Source File: RowTimeIntervalJoinTest.java From flink with Apache License 2.0 | 4 votes |
|
/** a.rowtime >= b.rowtime - 5 and a.rowtime <= b.rowtime + 9. **/
@Test
public void testRowTimeFullOuterJoin() throws Exception {
RowTimeIntervalJoin joinProcessFunc = new RowTimeIntervalJoin(
FlinkJoinType.FULL, -5, 9, 0, rowType, rowType, generatedFunction, 0, 0);
KeyedTwoInputStreamOperatorTestHarness<RowData, RowData, RowData, RowData> testHarness = createTestHarness(
joinProcessFunc);
testHarness.open();
testHarness.processElement1(insertRecord(1L, "k1"));
testHarness.processElement2(insertRecord(1L, "k2"));
assertEquals(2, testHarness.numEventTimeTimers());
assertEquals(4, testHarness.numKeyedStateEntries());
// The left row with timestamp = 1 will be padded and removed (14=1+5+1+((5+9)/2)).
testHarness.processWatermark1(new Watermark(14));
testHarness.processWatermark2(new Watermark(14));
assertEquals(1, testHarness.numEventTimeTimers());
assertEquals(2, testHarness.numKeyedStateEntries());
// The right row with timestamp = 1 will be padded and removed (18=1+9+1+((5+9)/2)).
testHarness.processWatermark1(new Watermark(18));
testHarness.processWatermark2(new Watermark(18));
assertEquals(0, testHarness.numEventTimeTimers());
assertEquals(0, testHarness.numKeyedStateEntries());
testHarness.processElement1(insertRecord(2L, "k1"));
testHarness.processElement2(insertRecord(2L, "k2"));
// The late rows with timestamp = 2 will not be cached, but a null padding result for the right
// row will be emitted.
assertEquals(0, testHarness.numKeyedStateEntries());
assertEquals(0, testHarness.numEventTimeTimers());
// Make sure the common (inner) join can be performed.
testHarness.processElement1(insertRecord(19L, "k1"));
testHarness.processElement1(insertRecord(20L, "k1"));
testHarness.processElement2(insertRecord(26L, "k1"));
testHarness.processElement2(insertRecord(25L, "k1"));
testHarness.processElement1(insertRecord(21L, "k1"));
testHarness.processElement2(insertRecord(39L, "k2"));
testHarness.processElement2(insertRecord(40L, "k2"));
testHarness.processElement1(insertRecord(50L, "k2"));
testHarness.processElement1(insertRecord(49L, "k2"));
testHarness.processElement2(insertRecord(41L, "k2"));
testHarness.processWatermark1(new Watermark(100));
testHarness.processWatermark2(new Watermark(100));
List<Object> expectedOutput = new ArrayList<>();
// The timestamp 14 is set with the triggered timer.
expectedOutput.add(insertRecord(1L, "k1", null, null));
expectedOutput.add(new Watermark(5));
// The timestamp 18 is set with the triggered timer.
expectedOutput.add(insertRecord(null, null, 1L, "k2"));
expectedOutput.add(new Watermark(9));
expectedOutput.add(insertRecord(2L, "k1", null, null));
expectedOutput.add(insertRecord(null, null, 2L, "k2"));
expectedOutput.add(insertRecord(20L, "k1", 25L, "k1"));
expectedOutput.add(insertRecord(21L, "k1", 25L, "k1"));
expectedOutput.add(insertRecord(21L, "k1", 26L, "k1"));
expectedOutput.add(insertRecord(49L, "k2", 40L, "k2"));
expectedOutput.add(insertRecord(49L, "k2", 41L, "k2"));
expectedOutput.add(insertRecord(50L, "k2", 41L, "k2"));
// The timestamp 32 is set with the triggered timer.
expectedOutput.add(insertRecord(19L, "k1", null, null));
// The timestamp 56 is set with the triggered timer.
expectedOutput.add(insertRecord(null, null, 39L, "k2"));
expectedOutput.add(new Watermark(91));
assertor.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
testHarness.close();
}
Example 14
| Source File: ProcTimeIntervalJoinTest.java From flink with Apache License 2.0 | 4 votes |
|
/** a.proctime >= b.proctime - 10 and a.proctime <= b.proctime + 20. **/
@Test
public void testProcTimeInnerJoinWithCommonBounds() throws Exception {
ProcTimeIntervalJoin joinProcessFunc = new ProcTimeIntervalJoin(
FlinkJoinType.INNER, -10, 20, rowType, rowType, generatedFunction);
KeyedTwoInputStreamOperatorTestHarness<RowData, RowData, RowData, RowData> testHarness = createTestHarness(
joinProcessFunc);
testHarness.open();
testHarness.setProcessingTime(1);
testHarness.processElement1(insertRecord(1L, "1a1"));
assertEquals(1, testHarness.numProcessingTimeTimers());
testHarness.setProcessingTime(2);
testHarness.processElement1(insertRecord(2L, "2a2"));
// timers for key = 1 and key = 2
assertEquals(2, testHarness.numProcessingTimeTimers());
testHarness.setProcessingTime(3);
testHarness.processElement1(insertRecord(1L, "1a3"));
assertEquals(4, testHarness.numKeyedStateEntries());
// The number of timers won't increase.
assertEquals(2, testHarness.numProcessingTimeTimers());
testHarness.processElement2(insertRecord(1L, "1b3"));
testHarness.setProcessingTime(4);
testHarness.processElement2(insertRecord(2L, "2b4"));
// The number of states should be doubled.
assertEquals(8, testHarness.numKeyedStateEntries());
assertEquals(4, testHarness.numProcessingTimeTimers());
// Test for -10 boundary (13 - 10 = 3).
// The left row (key = 1) with timestamp = 1 will be eagerly removed here.
testHarness.setProcessingTime(13);
testHarness.processElement2(insertRecord(1L, "1b13"));
// Test for +20 boundary (13 + 20 = 33).
testHarness.setProcessingTime(33);
assertEquals(4, testHarness.numKeyedStateEntries());
assertEquals(2, testHarness.numProcessingTimeTimers());
testHarness.processElement1(insertRecord(1L, "1a33"));
testHarness.processElement1(insertRecord(2L, "2a33"));
// The left row (key = 2) with timestamp = 2 will be eagerly removed here.
testHarness.processElement2(insertRecord(2L, "2b33"));
List<Object> expectedOutput = new ArrayList<>();
expectedOutput.add(insertRecord(1L, "1a1", 1L, "1b3"));
expectedOutput.add(insertRecord(1L, "1a3", 1L, "1b3"));
expectedOutput.add(insertRecord(2L, "2a2", 2L, "2b4"));
expectedOutput.add(insertRecord(1L, "1a3", 1L, "1b13"));
expectedOutput.add(insertRecord(1L, "1a33", 1L, "1b13"));
expectedOutput.add(insertRecord(2L, "2a33", 2L, "2b33"));
assertor.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
testHarness.close();
}
Example 15
| Source File: ProcTimeIntervalJoinTest.java From flink with Apache License 2.0 | 4 votes |
|
/** a.proctime >= b.proctime - 10 and a.proctime <= b.proctime - 5. **/
@Test
public void testProcTimeInnerJoinWithNegativeBounds() throws Exception {
ProcTimeIntervalJoin joinProcessFunc = new ProcTimeIntervalJoin(
FlinkJoinType.INNER, -10, -5, rowType, rowType, generatedFunction);
KeyedTwoInputStreamOperatorTestHarness<RowData, RowData, RowData, RowData> testHarness = createTestHarness(
joinProcessFunc);
testHarness.open();
testHarness.setProcessingTime(1);
testHarness.processElement1(insertRecord(1L, "1a1"));
testHarness.setProcessingTime(2);
testHarness.processElement1(insertRecord(2L, "2a2"));
testHarness.setProcessingTime(3);
testHarness.processElement1(insertRecord(1L, "1a3"));
assertEquals(4, testHarness.numKeyedStateEntries());
assertEquals(2, testHarness.numProcessingTimeTimers());
// All the right rows will not be cached.
testHarness.processElement2(insertRecord(1L, "1b3"));
assertEquals(4, testHarness.numKeyedStateEntries());
assertEquals(2, testHarness.numProcessingTimeTimers());
testHarness.setProcessingTime(7);
// Meets a.proctime <= b.proctime - 5.
// This row will only be joined without being cached (7 >= 7 - 5).
testHarness.processElement2(insertRecord(2L, "2b7"));
assertEquals(4, testHarness.numKeyedStateEntries());
assertEquals(2, testHarness.numProcessingTimeTimers());
testHarness.setProcessingTime(12);
// The left row (key = 1) with timestamp = 1 will be eagerly removed here.
testHarness.processElement2(insertRecord(1L, "1b12"));
// We add a delay (relativeWindowSize / 2) for cleaning up state.
// No timers will be triggered here.
testHarness.setProcessingTime(13);
assertEquals(4, testHarness.numKeyedStateEntries());
assertEquals(2, testHarness.numProcessingTimeTimers());
// Trigger the timer registered by the left row (key = 1) with timestamp = 1
// (1 + 10 + 2 + 0 + 1 = 14).
// The left row (key = 1) with timestamp = 3 will removed here.
testHarness.setProcessingTime(14);
assertEquals(2, testHarness.numKeyedStateEntries());
assertEquals(1, testHarness.numProcessingTimeTimers());
// Clean up the left row (key = 2) with timestamp = 2.
testHarness.setProcessingTime(16);
assertEquals(0, testHarness.numKeyedStateEntries());
assertEquals(0, testHarness.numProcessingTimeTimers());
List<Object> expectedOutput = new ArrayList<>();
expectedOutput.add(insertRecord(2L, "2a2", 2L, "2b7"));
expectedOutput.add(insertRecord(1L, "1a3", 1L, "1b12"));
assertor.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
testHarness.close();
}
