org.apache.flink.streaming.util.MockStreamingRuntimeContext Java Examples
The following examples show how to use
org.apache.flink.streaming.util.MockStreamingRuntimeContext.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: WatermarkTrackerTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void test() { long watermark = 0; TestWatermarkTracker ws = new TestWatermarkTracker(); ws.open(new MockStreamingRuntimeContext(false, 1, 0)); Assert.assertEquals(Long.MIN_VALUE, ws.updateWatermark(Long.MIN_VALUE)); Assert.assertEquals(Long.MIN_VALUE, ws.updateWatermark(watermark)); // timeout wm1 clock.add(WatermarkTracker.DEFAULT_UPDATE_TIMEOUT_MILLIS + 1); Assert.assertEquals(watermark, ws.updateWatermark(watermark)); Assert.assertEquals(watermark, ws.updateWatermark(watermark - 1)); // min watermark wm1.watermark = watermark + 1; wm1.lastUpdated = clock.longValue(); Assert.assertEquals(watermark, ws.updateWatermark(watermark)); Assert.assertEquals(watermark + 1, ws.updateWatermark(watermark + 1)); }
Example #2
Source File: WatermarkTrackerTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void test() { long watermark = 0; TestWatermarkTracker ws = new TestWatermarkTracker(); ws.open(new MockStreamingRuntimeContext(false, 1, 0)); Assert.assertEquals(Long.MIN_VALUE, ws.updateWatermark(Long.MIN_VALUE)); Assert.assertEquals(Long.MIN_VALUE, ws.updateWatermark(watermark)); // timeout wm1 clock.add(WatermarkTracker.DEFAULT_UPDATE_TIMEOUT_MILLIS + 1); Assert.assertEquals(watermark, ws.updateWatermark(watermark)); Assert.assertEquals(watermark, ws.updateWatermark(watermark - 1)); // min watermark wm1.watermark = watermark + 1; wm1.lastUpdated = clock.longValue(); Assert.assertEquals(watermark, ws.updateWatermark(watermark)); Assert.assertEquals(watermark + 1, ws.updateWatermark(watermark + 1)); }
Example #3
Source File: KeyedStateInputFormatTest.java From flink with Apache License 2.0 | 6 votes |
@Nonnull private List<Integer> readInputSplit(KeyGroupRangeInputSplit split, KeyedStateReaderFunction<Integer, Integer> userFunction) throws IOException { KeyedStateInputFormat<Integer, VoidNamespace, Integer> format = new KeyedStateInputFormat<>( new OperatorState(OperatorIDGenerator.fromUid("uid"), 1, 4), new MemoryStateBackend(), new Configuration(), new KeyedStateReaderOperator<>(userFunction, Types.INT)); List<Integer> data = new ArrayList<>(); format.setRuntimeContext(new MockStreamingRuntimeContext(false, 1, 0)); format.openInputFormat(); format.open(split); while (!format.reachedEnd()) { data.add(format.nextRecord(0)); } format.close(); format.closeInputFormat(); data.sort(Comparator.comparingInt(id -> id)); return data; }
Example #4
Source File: KeyedStateInputFormatTest.java From flink with Apache License 2.0 | 6 votes |
@Nonnull private List<Integer> readInputSplit(KeyGroupRangeInputSplit split, KeyedStateReaderFunction<Integer, Integer> userFunction) throws IOException { KeyedStateInputFormat<Integer, Integer> format = new KeyedStateInputFormat<>( new OperatorState(OperatorIDGenerator.fromUid("uid"), 1, 4), new MemoryStateBackend(), Types.INT, userFunction); List<Integer> data = new ArrayList<>(); format.setRuntimeContext(new MockStreamingRuntimeContext(false, 1, 0)); format.openInputFormat(); format.open(split); while (!format.reachedEnd()) { data.add(format.nextRecord(0)); } format.close(); format.closeInputFormat(); data.sort(Comparator.comparingInt(id -> id)); return data; }
Example #5
Source File: WatermarkTrackerTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void test() { long watermark = 0; TestWatermarkTracker ws = new TestWatermarkTracker(); ws.open(new MockStreamingRuntimeContext(false, 1, 0)); Assert.assertEquals(Long.MIN_VALUE, ws.updateWatermark(Long.MIN_VALUE)); Assert.assertEquals(Long.MIN_VALUE, ws.updateWatermark(watermark)); // timeout wm1 clock.add(WatermarkTracker.DEFAULT_UPDATE_TIMEOUT_MILLIS + 1); Assert.assertEquals(watermark, ws.updateWatermark(watermark)); Assert.assertEquals(watermark, ws.updateWatermark(watermark - 1)); // min watermark wm1.watermark = watermark + 1; wm1.lastUpdated = clock.longValue(); Assert.assertEquals(watermark, ws.updateWatermark(watermark)); Assert.assertEquals(watermark + 1, ws.updateWatermark(watermark + 1)); }
Example #6
Source File: CollectSinkFunctionTest.java From flink with Apache License 2.0 | 6 votes |
@Before public void before() throws Exception { ioManager = new IOManagerAsync(); MockEnvironment environment = new MockEnvironmentBuilder() .setTaskName("mockTask") .setManagedMemorySize(4 * MemoryManager.DEFAULT_PAGE_SIZE) .setIOManager(ioManager) .build(); runtimeContext = new MockStreamingRuntimeContext(false, 1, 0, environment); gateway = new MockOperatorEventGateway(); coordinator = new CollectSinkOperatorCoordinator(SOCKET_TIMEOUT_MILLIS); coordinator.start(); // only used in checkpointed tests functionInitializationContext = new MockFunctionInitializationContext(); jobFinished = false; }
Example #7
Source File: PrintSinkFunctionTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testPrintSinkStdOut() throws Exception { PrintSinkFunction<String> printSink = new PrintSinkFunction<>(); printSink.setRuntimeContext(new MockStreamingRuntimeContext(false, 1, 0)); printSink.open(new Configuration()); printSink.invoke("hello world!", SinkContextUtil.forTimestamp(0)); assertEquals("Print to System.out", printSink.toString()); assertEquals("hello world!" + line, arrayOutputStream.toString()); printSink.close(); }
Example #8
Source File: PrintSinkFunctionTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testPrintSinkStdErr() throws Exception { PrintSinkFunction<String> printSink = new PrintSinkFunction<>(true); printSink.setRuntimeContext(new MockStreamingRuntimeContext(false, 1, 0)); printSink.open(new Configuration()); printSink.invoke("hello world!", SinkContextUtil.forTimestamp(0)); assertEquals("Print to System.err", printSink.toString()); assertEquals("hello world!" + line, arrayErrorStream.toString()); printSink.close(); }
Example #9
Source File: UnionStateInputFormatTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testReadUnionOperatorState() throws Exception { try (OneInputStreamOperatorTestHarness<Integer, Void> testHarness = getTestHarness()) { testHarness.open(); testHarness.processElement(1, 0); testHarness.processElement(2, 0); testHarness.processElement(3, 0); OperatorSubtaskState subtaskState = testHarness.snapshot(0, 0); OperatorState state = new OperatorState(OperatorIDGenerator.fromUid("uid"), 1, 4); state.putState(0, subtaskState); OperatorStateInputSplit split = new OperatorStateInputSplit(subtaskState.getManagedOperatorState(), 0); UnionStateInputFormat<Integer> format = new UnionStateInputFormat<>(state, descriptor); format.setRuntimeContext(new MockStreamingRuntimeContext(false, 1, 0)); format.open(split); List<Integer> results = new ArrayList<>(); while (!format.reachedEnd()) { results.add(format.nextRecord(0)); } results.sort(Comparator.naturalOrder()); Assert.assertEquals("Failed to read correct list state from state backend", Arrays.asList(1, 2, 3), results); } }
Example #10
Source File: PrintSinkFunctionTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testPrintSinkWithPrefix() throws Exception { PrintSinkFunction<String> printSink = new PrintSinkFunction<>(); printSink.setRuntimeContext(new MockStreamingRuntimeContext(false, 2, 1)); printSink.open(new Configuration()); printSink.invoke("hello world!", SinkContextUtil.forTimestamp(0)); assertEquals("Print to System.out", printSink.toString()); assertEquals("2> hello world!" + line, arrayOutputStream.toString()); printSink.close(); }
Example #11
Source File: BroadcastStateInputFormatTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testReadBroadcastState() throws Exception { try (TwoInputStreamOperatorTestHarness<Void, Integer, Void> testHarness = getTestHarness()) { testHarness.open(); testHarness.processElement2(new StreamRecord<>(1)); testHarness.processElement2(new StreamRecord<>(2)); testHarness.processElement2(new StreamRecord<>(3)); OperatorSubtaskState subtaskState = testHarness.snapshot(0, 0); OperatorState state = new OperatorState(OperatorIDGenerator.fromUid("uid"), 1, 4); state.putState(0, subtaskState); OperatorStateInputSplit split = new OperatorStateInputSplit(subtaskState.getManagedOperatorState(), 0); BroadcastStateInputFormat<Integer, Integer> format = new BroadcastStateInputFormat<>(state, descriptor); format.setRuntimeContext(new MockStreamingRuntimeContext(false, 1, 0)); format.open(split); Map<Integer, Integer> results = new HashMap<>(3); while (!format.reachedEnd()) { Tuple2<Integer, Integer> entry = format.nextRecord(null); results.put(entry.f0, entry.f1); } Map<Integer, Integer> expected = new HashMap<>(3); expected.put(1, 1); expected.put(2, 2); expected.put(3, 3); Assert.assertEquals("Failed to read correct list state from state backend", expected, results); } }
Example #12
Source File: PrintSinkFunctionTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testPrintSinkWithIdentifierAndPrefix() throws Exception { PrintSinkFunction<String> printSink = new PrintSinkFunction<>("mySink", false); printSink.setRuntimeContext(new MockStreamingRuntimeContext(false, 2, 1)); printSink.open(new Configuration()); printSink.invoke("hello world!", SinkContextUtil.forTimestamp(0)); assertEquals("Print to System.out", printSink.toString()); assertEquals("mySink:2> hello world!" + line, arrayOutputStream.toString()); printSink.close(); }
Example #13
Source File: ListStateInputFormatTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testReadListOperatorState() throws Exception { try (OneInputStreamOperatorTestHarness<Integer, Void> testHarness = getTestHarness()) { testHarness.open(); testHarness.processElement(1, 0); testHarness.processElement(2, 0); testHarness.processElement(3, 0); OperatorSubtaskState subtaskState = testHarness.snapshot(0, 0); OperatorState state = new OperatorState(OperatorIDGenerator.fromUid("uid"), 1, 4); state.putState(0, subtaskState); OperatorStateInputSplit split = new OperatorStateInputSplit(subtaskState.getManagedOperatorState(), 0); ListStateInputFormat<Integer> format = new ListStateInputFormat<>(state, descriptor); format.setRuntimeContext(new MockStreamingRuntimeContext(false, 1, 0)); format.open(split); List<Integer> results = new ArrayList<>(); while (!format.reachedEnd()) { results.add(format.nextRecord(0)); } results.sort(Comparator.naturalOrder()); Assert.assertEquals( "Failed to read correct list state from state backend", Arrays.asList(1, 2, 3), results); } }
Example #14
Source File: FlinkKafkaConsumerBaseTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@SuppressWarnings("unchecked") private static <T, S> void setupConsumer( FlinkKafkaConsumerBase<T> consumer, boolean isRestored, ListState<S> restoredListState, boolean isCheckpointingEnabled, int subtaskIndex, int totalNumSubtasks) throws Exception { // run setup procedure in operator life cycle consumer.setRuntimeContext(new MockStreamingRuntimeContext(isCheckpointingEnabled, totalNumSubtasks, subtaskIndex)); consumer.initializeState(new MockFunctionInitializationContext(isRestored, new MockOperatorStateStore(restoredListState))); consumer.open(new Configuration()); }
Example #15
Source File: SavepointOutputFormatTest.java From flink with Apache License 2.0 | 5 votes |
private SavepointOutputFormat createSavepointOutputFormat(Path path) throws Exception { RuntimeContext ctx = new MockStreamingRuntimeContext(false, 1, 0); SavepointOutputFormat format = new SavepointOutputFormat(path); format.setRuntimeContext(ctx); return format; }
Example #16
Source File: PrintSinkFunctionTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testPrintSinkWithIdentifierButNoPrefix() throws Exception { PrintSinkFunction<String> printSink = new PrintSinkFunction<>("mySink", false); printSink.setRuntimeContext(new MockStreamingRuntimeContext(false, 1, 0)); printSink.open(new Configuration()); printSink.invoke("hello world!", SinkContextUtil.forTimestamp(0)); assertEquals("Print to System.out", printSink.toString()); assertEquals("mySink> hello world!" + line, arrayOutputStream.toString()); printSink.close(); }
Example #17
Source File: FlinkKafkaConsumerBaseTest.java From flink with Apache License 2.0 | 5 votes |
@SuppressWarnings("unchecked") private static <T, S> void setupConsumer( FlinkKafkaConsumerBase<T> consumer, boolean isRestored, ListState<S> restoredListState, boolean isCheckpointingEnabled, int subtaskIndex, int totalNumSubtasks) throws Exception { // run setup procedure in operator life cycle consumer.setRuntimeContext(new MockStreamingRuntimeContext(isCheckpointingEnabled, totalNumSubtasks, subtaskIndex)); consumer.initializeState(new MockFunctionInitializationContext(isRestored, new MockOperatorStateStore(restoredListState))); consumer.open(new Configuration()); }
Example #18
Source File: PrintSinkFunctionTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testPrintSinkWithIdentifierButNoPrefix() throws Exception { PrintSinkFunction<String> printSink = new PrintSinkFunction<>("mySink", false); printSink.setRuntimeContext(new MockStreamingRuntimeContext(false, 1, 0)); printSink.open(new Configuration()); printSink.invoke("hello world!", SinkContextUtil.forTimestamp(0)); assertEquals("Print to System.out", printSink.toString()); assertEquals("mySink> hello world!" + line, arrayOutputStream.toString()); printSink.close(); }
Example #19
Source File: PrintSinkFunctionTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testPrintSinkWithIdentifierAndPrefix() throws Exception { PrintSinkFunction<String> printSink = new PrintSinkFunction<>("mySink", false); printSink.setRuntimeContext(new MockStreamingRuntimeContext(false, 2, 1)); printSink.open(new Configuration()); printSink.invoke("hello world!", SinkContextUtil.forTimestamp(0)); assertEquals("Print to System.out", printSink.toString()); assertEquals("mySink:2> hello world!" + line, arrayOutputStream.toString()); printSink.close(); }
Example #20
Source File: PrintSinkFunctionTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testPrintSinkWithPrefix() throws Exception { PrintSinkFunction<String> printSink = new PrintSinkFunction<>(); printSink.setRuntimeContext(new MockStreamingRuntimeContext(false, 2, 1)); printSink.open(new Configuration()); printSink.invoke("hello world!", SinkContextUtil.forTimestamp(0)); assertEquals("Print to System.out", printSink.toString()); assertEquals("2> hello world!" + line, arrayOutputStream.toString()); printSink.close(); }
Example #21
Source File: PrintSinkFunctionTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testPrintSinkStdErr() throws Exception { PrintSinkFunction<String> printSink = new PrintSinkFunction<>(true); printSink.setRuntimeContext(new MockStreamingRuntimeContext(false, 1, 0)); printSink.open(new Configuration()); printSink.invoke("hello world!", SinkContextUtil.forTimestamp(0)); assertEquals("Print to System.err", printSink.toString()); assertEquals("hello world!" + line, arrayErrorStream.toString()); printSink.close(); }
Example #22
Source File: PrintSinkFunctionTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testPrintSinkStdOut() throws Exception { PrintSinkFunction<String> printSink = new PrintSinkFunction<>(); printSink.setRuntimeContext(new MockStreamingRuntimeContext(false, 1, 0)); printSink.open(new Configuration()); printSink.invoke("hello world!", SinkContextUtil.forTimestamp(0)); assertEquals("Print to System.out", printSink.toString()); assertEquals("hello world!" + line, arrayOutputStream.toString()); printSink.close(); }
Example #23
Source File: UnionStateInputFormatTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testReadUnionOperatorState() throws Exception { try (OneInputStreamOperatorTestHarness<Integer, Void> testHarness = getTestHarness()) { testHarness.open(); testHarness.processElement(1, 0); testHarness.processElement(2, 0); testHarness.processElement(3, 0); OperatorSubtaskState subtaskState = testHarness.snapshot(0, 0); OperatorState state = new OperatorState(OperatorIDGenerator.fromUid("uid"), 1, 4); state.putState(0, subtaskState); OperatorStateInputSplit split = new OperatorStateInputSplit(subtaskState.getManagedOperatorState(), 0); UnionStateInputFormat<Integer> format = new UnionStateInputFormat<>(state, descriptor); format.setRuntimeContext(new MockStreamingRuntimeContext(false, 1, 0)); format.open(split); List<Integer> results = new ArrayList<>(); while (!format.reachedEnd()) { results.add(format.nextRecord(0)); } results.sort(Comparator.naturalOrder()); Assert.assertEquals("Failed to read correct list state from state backend", Arrays.asList(1, 2, 3), results); } }
Example #24
Source File: BroadcastStateInputFormatTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testReadBroadcastState() throws Exception { try (TwoInputStreamOperatorTestHarness<Void, Integer, Void> testHarness = getTestHarness()) { testHarness.open(); testHarness.processElement2(new StreamRecord<>(1)); testHarness.processElement2(new StreamRecord<>(2)); testHarness.processElement2(new StreamRecord<>(3)); OperatorSubtaskState subtaskState = testHarness.snapshot(0, 0); OperatorState state = new OperatorState(OperatorIDGenerator.fromUid("uid"), 1, 4); state.putState(0, subtaskState); OperatorStateInputSplit split = new OperatorStateInputSplit(subtaskState.getManagedOperatorState(), 0); BroadcastStateInputFormat<Integer, Integer> format = new BroadcastStateInputFormat<>(state, descriptor); format.setRuntimeContext(new MockStreamingRuntimeContext(false, 1, 0)); format.open(split); Map<Integer, Integer> results = new HashMap<>(3); while (!format.reachedEnd()) { Tuple2<Integer, Integer> entry = format.nextRecord(null); results.put(entry.f0, entry.f1); } Map<Integer, Integer> expected = new HashMap<>(3); expected.put(1, 1); expected.put(2, 2); expected.put(3, 3); Assert.assertEquals("Failed to read correct list state from state backend", expected, results); } }
Example #25
Source File: ListStateInputFormatTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testReadListOperatorState() throws Exception { try (OneInputStreamOperatorTestHarness<Integer, Void> testHarness = getTestHarness()) { testHarness.open(); testHarness.processElement(1, 0); testHarness.processElement(2, 0); testHarness.processElement(3, 0); OperatorSubtaskState subtaskState = testHarness.snapshot(0, 0); OperatorState state = new OperatorState(OperatorIDGenerator.fromUid("uid"), 1, 4); state.putState(0, subtaskState); OperatorStateInputSplit split = new OperatorStateInputSplit(subtaskState.getManagedOperatorState(), 0); ListStateInputFormat<Integer> format = new ListStateInputFormat<>(state, descriptor); format.setRuntimeContext(new MockStreamingRuntimeContext(false, 1, 0)); format.open(split); List<Integer> results = new ArrayList<>(); while (!format.reachedEnd()) { results.add(format.nextRecord(0)); } results.sort(Comparator.naturalOrder()); Assert.assertEquals( "Failed to read correct list state from state backend", Arrays.asList(1, 2, 3), results); } }
Example #26
Source File: SavepointOutputFormatTest.java From flink with Apache License 2.0 | 5 votes |
private SavepointOutputFormat createSavepointOutputFormat(Path path) throws Exception { RuntimeContext ctx = new MockStreamingRuntimeContext(false, 1, 0); SavepointOutputFormat format = new SavepointOutputFormat(path); format.setRuntimeContext(ctx); return format; }
Example #27
Source File: FlinkKafkaConsumerBaseTest.java From flink with Apache License 2.0 | 5 votes |
@SuppressWarnings("unchecked") private static <T, S> void setupConsumer( FlinkKafkaConsumerBase<T> consumer, boolean isRestored, ListState<S> restoredListState, boolean isCheckpointingEnabled, int subtaskIndex, int totalNumSubtasks) throws Exception { // run setup procedure in operator life cycle consumer.setRuntimeContext(new MockStreamingRuntimeContext(isCheckpointingEnabled, totalNumSubtasks, subtaskIndex)); consumer.initializeState(new MockFunctionInitializationContext(isRestored, new MockOperatorStateStore(restoredListState))); consumer.open(new Configuration()); }
Example #28
Source File: PrintSinkFunctionTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testPrintSinkWithIdentifierButNoPrefix() throws Exception { PrintSinkFunction<String> printSink = new PrintSinkFunction<>("mySink", false); printSink.setRuntimeContext(new MockStreamingRuntimeContext(false, 1, 0)); printSink.open(new Configuration()); printSink.invoke("hello world!", SinkContextUtil.forTimestamp(0)); assertEquals("Print to System.out", printSink.toString()); assertEquals("mySink> hello world!" + line, arrayOutputStream.toString()); printSink.close(); }
Example #29
Source File: PrintSinkFunctionTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testPrintSinkWithIdentifierAndPrefix() throws Exception { PrintSinkFunction<String> printSink = new PrintSinkFunction<>("mySink", false); printSink.setRuntimeContext(new MockStreamingRuntimeContext(false, 2, 1)); printSink.open(new Configuration()); printSink.invoke("hello world!", SinkContextUtil.forTimestamp(0)); assertEquals("Print to System.out", printSink.toString()); assertEquals("mySink:2> hello world!" + line, arrayOutputStream.toString()); printSink.close(); }
Example #30
Source File: PrintSinkFunctionTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testPrintSinkWithPrefix() throws Exception { PrintSinkFunction<String> printSink = new PrintSinkFunction<>(); printSink.setRuntimeContext(new MockStreamingRuntimeContext(false, 2, 1)); printSink.open(new Configuration()); printSink.invoke("hello world!", SinkContextUtil.forTimestamp(0)); assertEquals("Print to System.out", printSink.toString()); assertEquals("2> hello world!" + line, arrayOutputStream.toString()); printSink.close(); }