Java Code Examples for org.apache.flink.streaming.util.KeyedTwoInputStreamOperatorTestHarness#setProcessingTime()

The following examples show how to use org.apache.flink.streaming.util.KeyedTwoInputStreamOperatorTestHarness#setProcessingTime() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: UrlDBFunctionTest.java    From flink-crawler with Apache License 2.0 5 votes vote down vote up 
private void setProcessingTime(long newTime) throws Exception {
    for (int subTaskIndex = 0; subTaskIndex < _testHarnesses.length; subTaskIndex++) {
        KeyedTwoInputStreamOperatorTestHarness<String, CrawlStateUrl, DomainScore, FetchUrl> testHarness = _testHarnesses[subTaskIndex];
        LOGGER.debug("Test harness ({}/{}) processing time is now {}", subTaskIndex + 1,
                _testHarnesses.length, newTime);
        testHarness.setProcessingTime(newTime);
    }
}
Example 2
Source File: UrlDBFunctionTest.java    From flink-crawler with Apache License 2.0 5 votes vote down vote up 
private void addProcessingTime(long extraTime) throws Exception {
    for (int subTaskIndex = 0; subTaskIndex < _testHarnesses.length; subTaskIndex++) {
        KeyedTwoInputStreamOperatorTestHarness<String, CrawlStateUrl, DomainScore, FetchUrl> testHarness = _testHarnesses[subTaskIndex];
        long newTime = testHarness.getProcessingTime() + extraTime;
        LOGGER.debug("Test harness ({}/{}) processing time is now {}", subTaskIndex + 1,
                _testHarnesses.length, newTime);
        testHarness.setProcessingTime(newTime);
    }
}
Example 3
Source File: ProcTimeBoundedStreamJoinTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
/** a.proctime >= b.proctime - 10 and a.proctime <= b.proctime + 20. **/
@Test
public void testProcTimeInnerJoinWithCommonBounds() throws Exception {
	ProcTimeBoundedStreamJoin joinProcessFunc = new ProcTimeBoundedStreamJoin(
			FlinkJoinType.INNER, -10, 20, rowType, rowType, generatedFunction);
	KeyedTwoInputStreamOperatorTestHarness<BaseRow, BaseRow, BaseRow, BaseRow> testHarness = createTestHarness(
			joinProcessFunc);
	testHarness.open();
	testHarness.setProcessingTime(1);
	testHarness.processElement1(record(1L, "1a1"));
	assertEquals(1, testHarness.numProcessingTimeTimers());

	testHarness.setProcessingTime(2);
	testHarness.processElement1(record(2L, "2a2"));
	// timers for key = 1 and key = 2
	assertEquals(2, testHarness.numProcessingTimeTimers());

	testHarness.setProcessingTime(3);
	testHarness.processElement1(record(1L, "1a3"));
	assertEquals(4, testHarness.numKeyedStateEntries());
	// The number of timers won't increase.
	assertEquals(2, testHarness.numProcessingTimeTimers());

	testHarness.processElement2(record(1L, "1b3"));

	testHarness.setProcessingTime(4);
	testHarness.processElement2(record(2L, "2b4"));
	// The number of states should be doubled.
	assertEquals(8, testHarness.numKeyedStateEntries());
	assertEquals(4, testHarness.numProcessingTimeTimers());

	// Test for -10 boundary (13 - 10 = 3).
	// The left row (key = 1) with timestamp = 1 will be eagerly removed here.
	testHarness.setProcessingTime(13);
	testHarness.processElement2(record(1L, "1b13"));

	// Test for +20 boundary (13 + 20 = 33).
	testHarness.setProcessingTime(33);
	assertEquals(4, testHarness.numKeyedStateEntries());
	assertEquals(2, testHarness.numProcessingTimeTimers());

	testHarness.processElement1(record(1L, "1a33"));
	testHarness.processElement1(record(2L, "2a33"));
	// The left row (key = 2) with timestamp = 2 will be eagerly removed here.
	testHarness.processElement2(record(2L, "2b33"));

	List<Object> expectedOutput = new ArrayList<>();
	expectedOutput.add(record(1L, "1a1", 1L, "1b3"));
	expectedOutput.add(record(1L, "1a3", 1L, "1b3"));
	expectedOutput.add(record(2L, "2a2", 2L, "2b4"));
	expectedOutput.add(record(1L, "1a3", 1L, "1b13"));
	expectedOutput.add(record(1L, "1a33", 1L, "1b13"));
	expectedOutput.add(record(2L, "2a33", 2L, "2b33"));

	assertor.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
	testHarness.close();
}
Example 4
Source File: ProcTimeBoundedStreamJoinTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
/** a.proctime >= b.proctime - 10 and a.proctime <= b.proctime - 5. **/
@Test
public void testProcTimeInnerJoinWithNegativeBounds() throws Exception {
	ProcTimeBoundedStreamJoin joinProcessFunc = new ProcTimeBoundedStreamJoin(
			FlinkJoinType.INNER, -10, -5, rowType, rowType, generatedFunction);

	KeyedTwoInputStreamOperatorTestHarness<BaseRow, BaseRow, BaseRow, BaseRow> testHarness = createTestHarness(
			joinProcessFunc);
	testHarness.open();

	testHarness.setProcessingTime(1);
	testHarness.processElement1(record(1L, "1a1"));

	testHarness.setProcessingTime(2);
	testHarness.processElement1(record(2L, "2a2"));

	testHarness.setProcessingTime(3);
	testHarness.processElement1(record(1L, "1a3"));
	assertEquals(4, testHarness.numKeyedStateEntries());
	assertEquals(2, testHarness.numProcessingTimeTimers());

	// All the right rows will not be cached.
	testHarness.processElement2(record(1L, "1b3"));
	assertEquals(4, testHarness.numKeyedStateEntries());
	assertEquals(2, testHarness.numProcessingTimeTimers());

	testHarness.setProcessingTime(7);

	// Meets a.proctime <= b.proctime - 5.
	// This row will only be joined without being cached (7 >= 7 - 5).
	testHarness.processElement2(record(2L, "2b7"));
	assertEquals(4, testHarness.numKeyedStateEntries());
	assertEquals(2, testHarness.numProcessingTimeTimers());

	testHarness.setProcessingTime(12);
	// The left row (key = 1) with timestamp = 1 will be eagerly removed here.
	testHarness.processElement2(record(1L, "1b12"));

	// We add a delay (relativeWindowSize / 2) for cleaning up state.
	// No timers will be triggered here.
	testHarness.setProcessingTime(13);
	assertEquals(4, testHarness.numKeyedStateEntries());
	assertEquals(2, testHarness.numProcessingTimeTimers());

	// Trigger the timer registered by the left row (key = 1) with timestamp = 1
	// (1 + 10 + 2 + 0 + 1 = 14).
	// The left row (key = 1) with timestamp = 3 will removed here.
	testHarness.setProcessingTime(14);
	assertEquals(2, testHarness.numKeyedStateEntries());
	assertEquals(1, testHarness.numProcessingTimeTimers());

	// Clean up the left row (key = 2) with timestamp = 2.
	testHarness.setProcessingTime(16);
	assertEquals(0, testHarness.numKeyedStateEntries());
	assertEquals(0, testHarness.numProcessingTimeTimers());

	List<Object> expectedOutput = new ArrayList<>();
	expectedOutput.add(record(2L, "2a2", 2L, "2b7"));
	expectedOutput.add(record(1L, "1a3", 1L, "1b12"));

	assertor.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
	testHarness.close();
}
Example 5
Source File: ProcTimeIntervalJoinTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
/** a.proctime >= b.proctime - 10 and a.proctime <= b.proctime + 20. **/
@Test
public void testProcTimeInnerJoinWithCommonBounds() throws Exception {
	ProcTimeIntervalJoin joinProcessFunc = new ProcTimeIntervalJoin(
			FlinkJoinType.INNER, -10, 20, rowType, rowType, generatedFunction);
	KeyedTwoInputStreamOperatorTestHarness<RowData, RowData, RowData, RowData> testHarness = createTestHarness(
			joinProcessFunc);
	testHarness.open();
	testHarness.setProcessingTime(1);
	testHarness.processElement1(insertRecord(1L, "1a1"));
	assertEquals(1, testHarness.numProcessingTimeTimers());

	testHarness.setProcessingTime(2);
	testHarness.processElement1(insertRecord(2L, "2a2"));
	// timers for key = 1 and key = 2
	assertEquals(2, testHarness.numProcessingTimeTimers());

	testHarness.setProcessingTime(3);
	testHarness.processElement1(insertRecord(1L, "1a3"));
	assertEquals(4, testHarness.numKeyedStateEntries());
	// The number of timers won't increase.
	assertEquals(2, testHarness.numProcessingTimeTimers());

	testHarness.processElement2(insertRecord(1L, "1b3"));

	testHarness.setProcessingTime(4);
	testHarness.processElement2(insertRecord(2L, "2b4"));
	// The number of states should be doubled.
	assertEquals(8, testHarness.numKeyedStateEntries());
	assertEquals(4, testHarness.numProcessingTimeTimers());

	// Test for -10 boundary (13 - 10 = 3).
	// The left row (key = 1) with timestamp = 1 will be eagerly removed here.
	testHarness.setProcessingTime(13);
	testHarness.processElement2(insertRecord(1L, "1b13"));

	// Test for +20 boundary (13 + 20 = 33).
	testHarness.setProcessingTime(33);
	assertEquals(4, testHarness.numKeyedStateEntries());
	assertEquals(2, testHarness.numProcessingTimeTimers());

	testHarness.processElement1(insertRecord(1L, "1a33"));
	testHarness.processElement1(insertRecord(2L, "2a33"));
	// The left row (key = 2) with timestamp = 2 will be eagerly removed here.
	testHarness.processElement2(insertRecord(2L, "2b33"));

	List<Object> expectedOutput = new ArrayList<>();
	expectedOutput.add(insertRecord(1L, "1a1", 1L, "1b3"));
	expectedOutput.add(insertRecord(1L, "1a3", 1L, "1b3"));
	expectedOutput.add(insertRecord(2L, "2a2", 2L, "2b4"));
	expectedOutput.add(insertRecord(1L, "1a3", 1L, "1b13"));
	expectedOutput.add(insertRecord(1L, "1a33", 1L, "1b13"));
	expectedOutput.add(insertRecord(2L, "2a33", 2L, "2b33"));

	assertor.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
	testHarness.close();
}
Example 6
Source File: ProcTimeIntervalJoinTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
/** a.proctime >= b.proctime - 10 and a.proctime <= b.proctime - 5. **/
@Test
public void testProcTimeInnerJoinWithNegativeBounds() throws Exception {
	ProcTimeIntervalJoin joinProcessFunc = new ProcTimeIntervalJoin(
			FlinkJoinType.INNER, -10, -5, rowType, rowType, generatedFunction);

	KeyedTwoInputStreamOperatorTestHarness<RowData, RowData, RowData, RowData> testHarness = createTestHarness(
			joinProcessFunc);
	testHarness.open();

	testHarness.setProcessingTime(1);
	testHarness.processElement1(insertRecord(1L, "1a1"));

	testHarness.setProcessingTime(2);
	testHarness.processElement1(insertRecord(2L, "2a2"));

	testHarness.setProcessingTime(3);
	testHarness.processElement1(insertRecord(1L, "1a3"));
	assertEquals(4, testHarness.numKeyedStateEntries());
	assertEquals(2, testHarness.numProcessingTimeTimers());

	// All the right rows will not be cached.
	testHarness.processElement2(insertRecord(1L, "1b3"));
	assertEquals(4, testHarness.numKeyedStateEntries());
	assertEquals(2, testHarness.numProcessingTimeTimers());

	testHarness.setProcessingTime(7);

	// Meets a.proctime <= b.proctime - 5.
	// This row will only be joined without being cached (7 >= 7 - 5).
	testHarness.processElement2(insertRecord(2L, "2b7"));
	assertEquals(4, testHarness.numKeyedStateEntries());
	assertEquals(2, testHarness.numProcessingTimeTimers());

	testHarness.setProcessingTime(12);
	// The left row (key = 1) with timestamp = 1 will be eagerly removed here.
	testHarness.processElement2(insertRecord(1L, "1b12"));

	// We add a delay (relativeWindowSize / 2) for cleaning up state.
	// No timers will be triggered here.
	testHarness.setProcessingTime(13);
	assertEquals(4, testHarness.numKeyedStateEntries());
	assertEquals(2, testHarness.numProcessingTimeTimers());

	// Trigger the timer registered by the left row (key = 1) with timestamp = 1
	// (1 + 10 + 2 + 0 + 1 = 14).
	// The left row (key = 1) with timestamp = 3 will removed here.
	testHarness.setProcessingTime(14);
	assertEquals(2, testHarness.numKeyedStateEntries());
	assertEquals(1, testHarness.numProcessingTimeTimers());

	// Clean up the left row (key = 2) with timestamp = 2.
	testHarness.setProcessingTime(16);
	assertEquals(0, testHarness.numKeyedStateEntries());
	assertEquals(0, testHarness.numProcessingTimeTimers());

	List<Object> expectedOutput = new ArrayList<>();
	expectedOutput.add(insertRecord(2L, "2a2", 2L, "2b7"));
	expectedOutput.add(insertRecord(1L, "1a3", 1L, "1b12"));

	assertor.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
	testHarness.close();
}