Java Code Examples for org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness#open()

/**
 * This test is meant to assure that testAtLeastOnceProducer is valid by testing that if flushing is disabled,
 * the snapshot method does indeed finishes without waiting for pending records;
 * we set a timeout because the test will not finish if the logic is broken.
 */
@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testDoesNotWaitForPendingRecordsIfFlushingDisabled() throws Throwable {
	final DummyFlinkKafkaProducer<String> producer = new DummyFlinkKafkaProducer<>(
		FakeStandardProducerConfig.get(), new KeyedSerializationSchemaWrapper<>(new SimpleStringSchema()), null);
	producer.setFlushOnCheckpoint(false);

	final KafkaProducer<?, ?> mockProducer = producer.getMockKafkaProducer();

	final OneInputStreamOperatorTestHarness<String, Object> testHarness =
		new OneInputStreamOperatorTestHarness<>(new StreamSink<>(producer));

	testHarness.open();

	testHarness.processElement(new StreamRecord<>("msg"));

	// make sure that all callbacks have not been completed
	verify(mockProducer, times(1)).send(any(ProducerRecord.class), any(Callback.class));

	// should return even if there are pending records
	testHarness.snapshot(123L, 123L);

	testHarness.close();
}
 

@Test
public void testOnElementCalledPerWindow() throws Exception {

	WindowAssigner<TimeWindow> mockAssigner = mockTimeWindowAssigner();
	Trigger<TimeWindow> mockTrigger = mockTrigger();
	NamespaceAggsHandleFunction<TimeWindow> mockAggregate = mockAggsHandleFunction();

	OneInputStreamOperatorTestHarness<BaseRow, BaseRow> testHarness =
			createWindowOperator(mockAssigner, mockTrigger, mockAggregate, 0L);

	testHarness.open();

	when(mockAssigner.assignWindows(anyGenericRow(), anyLong()))
			.thenReturn(Arrays.asList(new TimeWindow(2, 4), new TimeWindow(0, 2)));

	testHarness.processElement(record("String", 42, 1L));

	verify(mockTrigger).onElement(eq(baserow("String", 42, 1L)), eq(1L), eq(new TimeWindow(2, 4)));
	verify(mockTrigger).onElement(eq(baserow("String", 42, 1L)), eq(1L), eq(new TimeWindow(0, 2)));
	verify(mockTrigger, times(2)).onElement(any(), anyLong(), anyTimeWindow());
}
 

private void testTimeoutExceptionHandling(AsyncDataStream.OutputMode outputMode) throws Exception {
	OneInputStreamOperatorTestHarness<Integer, Integer> harness =
		createTestHarness(new NoOpAsyncFunction<>(), 10L, 2, outputMode);

	harness.getEnvironment().setExpectedExternalFailureCause(Throwable.class);
	harness.open();

	synchronized (harness.getCheckpointLock()) {
		harness.processElement(1, 1L);
	}

	harness.setProcessingTime(10L);

	synchronized (harness.getCheckpointLock()) {
		harness.close();
	}
}
 

@Test
public void testWithoutGenerateUpdateBefore() throws Exception {
	DeduplicateKeepLastRowFunction func = createFunction(false, true);
	OneInputStreamOperatorTestHarness<RowData, RowData> testHarness = createTestHarness(func);
	testHarness.open();
	testHarness.processElement(insertRecord("book", 1L, 12));
	testHarness.processElement(insertRecord("book", 2L, 11));
	testHarness.processElement(insertRecord("book", 1L, 13));
	testHarness.close();

	List<Object> expectedOutput = new ArrayList<>();
	// we do not output INSERT if generateUpdateBefore is false for deduplication last row
	expectedOutput.add(insertRecord("book", 1L, 12));
	expectedOutput.add(insertRecord("book", 2L, 11));
	expectedOutput.add(updateAfterRecord("book", 1L, 13));
	assertor.assertOutputEqualsSorted("output wrong.", expectedOutput, testHarness.getOutput());
}
 

@Test
public void testMap() throws Exception {
	StreamMap<Integer, String> operator = new StreamMap<Integer, String>(new Map());

	OneInputStreamOperatorTestHarness<Integer, String> testHarness = new OneInputStreamOperatorTestHarness<Integer, String>(operator);

	long initialTime = 0L;
	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>();

	testHarness.open();

	testHarness.processElement(new StreamRecord<Integer>(1, initialTime + 1));
	testHarness.processElement(new StreamRecord<Integer>(2, initialTime + 2));
	testHarness.processWatermark(new Watermark(initialTime + 2));
	testHarness.processElement(new StreamRecord<Integer>(3, initialTime + 3));

	expectedOutput.add(new StreamRecord<String>("+2", initialTime + 1));
	expectedOutput.add(new StreamRecord<String>("+3", initialTime + 2));
	expectedOutput.add(new Watermark(initialTime + 2));
	expectedOutput.add(new StreamRecord<String>("+4", initialTime + 3));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}
 

@Test
public void testTemporalLeftJoinWithFilter() throws Exception {
	OneInputStreamOperatorTestHarness<RowData, RowData> testHarness = createHarness(
		JoinType.LEFT_JOIN,
		FilterOnTable.WITH_FILTER);

	testHarness.open();

	testHarness.processElement(insertRecord(1, "a"));
	testHarness.processElement(insertRecord(2, "b"));
	testHarness.processElement(insertRecord(3, "c"));
	testHarness.processElement(insertRecord(4, "d"));
	testHarness.processElement(insertRecord(5, "e"));

	List<Object> expectedOutput = new ArrayList<>();
	expectedOutput.add(insertRecord(1, "a", 1, "Julian"));
	expectedOutput.add(insertRecord(2, "b", null, null));
	expectedOutput.add(insertRecord(3, "c", 3, "Jackson"));
	expectedOutput.add(insertRecord(4, "d", 4, "Fabian"));
	expectedOutput.add(insertRecord(5, "e", null, null));

	assertor.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
	testHarness.close();
}
 

@Test
public void testDisableGenerateUpdateBeforeWithRowNumber() throws Exception {
	AbstractTopNFunction func = createFunction(RankType.ROW_NUMBER, new ConstantRankRange(1, 2), false,
			true);
	OneInputStreamOperatorTestHarness<RowData, RowData> testHarness = createTestHarness(func);
	testHarness.open();
	testHarness.processElement(insertRecord("book", 1L, 12));
	testHarness.processElement(insertRecord("book", 2L, 19));
	testHarness.processElement(insertRecord("book", 4L, 11));
	testHarness.processElement(insertRecord("fruit", 4L, 33));
	testHarness.processElement(insertRecord("fruit", 3L, 44));
	testHarness.processElement(insertRecord("fruit", 5L, 22));
	testHarness.close();

	List<Object> expectedOutput = new ArrayList<>();
	// ("book", 1L, 12)
	// ("book", 2L, 19)
	expectedOutput.add(insertRecord("book", 1L, 12, 1L));
	expectedOutput.add(insertRecord("book", 2L, 19, 2L));
	// ("book", 4L, 11)
	expectedOutput.add(updateAfterRecord("book", 4L, 11, 1L));
	expectedOutput.add(updateAfterRecord("book", 1L, 12, 2L));
	// ("fruit", 4L, 33)
	// ("fruit", 3L, 44)
	expectedOutput.add(insertRecord("fruit", 4L, 33, 1L));
	expectedOutput.add(insertRecord("fruit", 3L, 44, 2L));
	// ("fruit", 5L, 22)
	expectedOutput.add(updateAfterRecord("fruit", 5L, 22, 1L));
	expectedOutput.add(updateAfterRecord("fruit", 4L, 33, 2L));
	assertorWithRowNumber
			.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
}
 

/**
 * Verifies that pipelines including {@link CoGroupedStreams} can be checkpointed properly,
 * which includes snapshotting configurations of any involved serializers.
 *
 * @see <a href="https://issues.apache.org/jira/browse/FLINK-6808">FLINK-6808</a>
 */
@Test
public void testCoGroupOperatorWithCheckpoint() throws Exception {

	// generate an operator for the co-group operation
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
	env.setParallelism(1);

	DataStream<Tuple2<String, Integer>> source1 = env.fromElements(Tuple2.of("a", 0), Tuple2.of("b", 3));
	DataStream<Tuple2<String, Integer>> source2 = env.fromElements(Tuple2.of("a", 1), Tuple2.of("b", 6));

	DataStream<String> coGroupWindow = source1.coGroup(source2)
		.where(new Tuple2KeyExtractor())
		.equalTo(new Tuple2KeyExtractor())
		.window(TumblingEventTimeWindows.of(Time.of(3, TimeUnit.MILLISECONDS)))
		.apply(new CoGroupFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, String>() {
			@Override
			public void coGroup(Iterable<Tuple2<String, Integer>> first,
								Iterable<Tuple2<String, Integer>> second,
								Collector<String> out) throws Exception {
				out.collect(first + ":" + second);
			}
		});

	OneInputTransformation<Tuple2<String, Integer>, String> transform = (OneInputTransformation<Tuple2<String, Integer>, String>) coGroupWindow.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, String> operator = transform.getOperator();

	// wrap the operator in the test harness, and perform a snapshot
	OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, String> testHarness =
		new KeyedOneInputStreamOperatorTestHarness<>(operator, new Tuple2KeyExtractor(), BasicTypeInfo.STRING_TYPE_INFO);

	testHarness.open();
	testHarness.snapshot(0L, 0L);
}
 

@Test
public void testFlatMap() throws Exception {
	StreamFlatMap<Integer, Integer> operator = new StreamFlatMap<Integer, Integer>(new MyFlatMap());

	OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new OneInputStreamOperatorTestHarness<Integer, Integer>(operator);

	long initialTime = 0L;
	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>();

	testHarness.open();

	testHarness.processElement(new StreamRecord<Integer>(1, initialTime + 1));
	testHarness.processElement(new StreamRecord<Integer>(2, initialTime + 2));
	testHarness.processWatermark(new Watermark(initialTime + 2));
	testHarness.processElement(new StreamRecord<Integer>(3, initialTime + 3));
	testHarness.processElement(new StreamRecord<Integer>(4, initialTime + 4));
	testHarness.processElement(new StreamRecord<Integer>(5, initialTime + 5));
	testHarness.processElement(new StreamRecord<Integer>(6, initialTime + 6));
	testHarness.processElement(new StreamRecord<Integer>(7, initialTime + 7));
	testHarness.processElement(new StreamRecord<Integer>(8, initialTime + 8));

	expectedOutput.add(new StreamRecord<Integer>(2, initialTime + 2));
	expectedOutput.add(new StreamRecord<Integer>(4, initialTime + 2));
	expectedOutput.add(new Watermark(initialTime + 2));
	expectedOutput.add(new StreamRecord<Integer>(4, initialTime + 4));
	expectedOutput.add(new StreamRecord<Integer>(16, initialTime + 4));
	expectedOutput.add(new StreamRecord<Integer>(6, initialTime + 6));
	expectedOutput.add(new StreamRecord<Integer>(36, initialTime + 6));
	expectedOutput.add(new StreamRecord<Integer>(8, initialTime + 8));
	expectedOutput.add(new StreamRecord<Integer>(64, initialTime + 8));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}
 

@Test
public void testRestore() throws Exception {
	final File outDir = tempFolder.newFolder();

	ValidatingBucketingSink<String> sink = (ValidatingBucketingSink<String>)
			new ValidatingBucketingSink<String>(outDir.getAbsolutePath(), expectedBucketFilesPrefix)
		.setWriter(new StringWriter<String>())
		.setBatchSize(5)
		.setPartPrefix(PART_PREFIX)
		.setInProgressPrefix("")
		.setPendingPrefix("")
		.setValidLengthPrefix("")
		.setInProgressSuffix(IN_PROGRESS_SUFFIX)
		.setPendingSuffix(PENDING_SUFFIX)
		.setValidLengthSuffix(VALID_LENGTH_SUFFIX)
		.setUseTruncate(false); // don't use truncate because files do not exist

	OneInputStreamOperatorTestHarness<String, Object> testHarness = new OneInputStreamOperatorTestHarness<>(
		new StreamSink<>(sink), 10, 1, 0);
	testHarness.setup();

	testHarness.initializeState(
		OperatorSnapshotUtil.getResourceFilename(
			"bucketing-sink-migration-test-flink" + testMigrateVersion + "-snapshot"));

	testHarness.open();

	assertTrue(sink.initializeCalled);

	testHarness.processElement(new StreamRecord<>("test1", 0L));
	testHarness.processElement(new StreamRecord<>("test2", 0L));

	checkLocalFs(outDir, 1, 1, 0, 0);

	testHarness.close();
}
 

/**
 * Tests TimeEvictor evictAfter behavior.
 */
@Test
public void testTimeEvictorEvictAfter() throws Exception {
	AtomicInteger closeCalled = new AtomicInteger(0);
	final int triggerCount = 2;
	final boolean evictAfter = true;

	@SuppressWarnings({"unchecked", "rawtypes"})
	TypeSerializer<StreamRecord<Tuple2<String, Integer>>> streamRecordSerializer =
		(TypeSerializer<StreamRecord<Tuple2<String, Integer>>>) new StreamElementSerializer(STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));

	ListStateDescriptor<StreamRecord<Tuple2<String, Integer>>> stateDesc =
		new ListStateDescriptor<>("window-contents", streamRecordSerializer);

	EvictingWindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, GlobalWindow> operator = new EvictingWindowOperator<>(
		GlobalWindows.create(),
		new GlobalWindow.Serializer(),
		new TupleKeySelector(),
		BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
		stateDesc,
		new InternalIterableWindowFunction<>(new RichSumReducer<GlobalWindow>(closeCalled)),
		CountTrigger.of(triggerCount),
		TimeEvictor.of(Time.seconds(2), evictAfter),
		0,
		null /* late data output tag */);

	OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
		new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);

	long initialTime = 0L;
	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	testHarness.open();

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1000));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 4000));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 20));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 999));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 3500));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 2001));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1001));

	expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 2), Long.MAX_VALUE));
	expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 2), Long.MAX_VALUE));
	expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 3), Long.MAX_VALUE));

	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new ResultSortComparator());

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 10999));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1002));

	expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 4), Long.MAX_VALUE));
	expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 5), Long.MAX_VALUE));

	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new ResultSortComparator());

	testHarness.close();

	Assert.assertEquals("Close was not called.", 1, closeCalled.get());

}
 

private void testPartFilesWithStringBucketer(
		OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Object> testHarness,
		File outDir,
		String partFileName1,
		String partFileName2) throws Exception {

	testHarness.setup();
	testHarness.open();

	// this creates a new bucket "test1" and part-0-0
	testHarness.processElement(new StreamRecord<>(Tuple2.of("test1", 1), 1L));
	TestUtils.checkLocalFs(outDir, 1, 0);

	// we take a checkpoint so we roll.
	testHarness.snapshot(1L, 1L);

	// these will close part-0-0 and open part-0-1
	testHarness.processElement(new StreamRecord<>(Tuple2.of("test1", 2), 2L));
	testHarness.processElement(new StreamRecord<>(Tuple2.of("test1", 3), 3L));

	// we take a checkpoint so we roll again.
	testHarness.snapshot(2L, 2L);

	TestUtils.checkLocalFs(outDir, 2, 0);

	Map<File, String> contents = TestUtils.getFileContentByPath(outDir);
	int fileCounter = 0;
	for (Map.Entry<File, String> fileContents : contents.entrySet()) {
		if (fileContents.getKey().getName().contains(partFileName1)) {
			fileCounter++;
			Assert.assertEquals("test1@1\n", fileContents.getValue());
		} else if (fileContents.getKey().getName().contains(partFileName2)) {
			fileCounter++;
			Assert.assertEquals("test1@2\ntest1@3\n", fileContents.getValue());
		}
		// check bucket name
		Assert.assertEquals("test1", fileContents.getKey().getParentFile().getName());
	}
	Assert.assertEquals(2L, fileCounter);

	// we acknowledge the latest checkpoint, so everything should be published.
	testHarness.notifyOfCompletedCheckpoint(2L);

	TestUtils.checkLocalFs(outDir, 0, 2);
}
 

/**
 * This tests user defined hdfs configuration.
 * @throws Exception
 */
@Test
public void testUserDefinedConfiguration() throws Exception {
	final String outPath = hdfsURI + "/string-non-rolling-with-config";
	final int numElements = 20;

	Map<String, String> properties = new HashMap<>();
	Schema keySchema = Schema.create(Schema.Type.INT);
	Schema valueSchema = Schema.create(Schema.Type.STRING);
	properties.put(AvroKeyValueSinkWriter.CONF_OUTPUT_KEY_SCHEMA, keySchema.toString());
	properties.put(AvroKeyValueSinkWriter.CONF_OUTPUT_VALUE_SCHEMA, valueSchema.toString());
	properties.put(AvroKeyValueSinkWriter.CONF_COMPRESS, String.valueOf(true));
	properties.put(AvroKeyValueSinkWriter.CONF_COMPRESS_CODEC, DataFileConstants.SNAPPY_CODEC);

	Configuration conf = new Configuration();
	conf.set("io.file.buffer.size", "40960");

	BucketingSink<Tuple2<Integer, String>> sink = new BucketingSink<Tuple2<Integer, String>>(outPath)
		.setFSConfig(conf)
		.setWriter(new StreamWriterWithConfigCheck<Integer, String>(properties, "io.file.buffer.size", "40960"))
		.setBucketer(new BasePathBucketer<Tuple2<Integer, String>>())
		.setPartPrefix(PART_PREFIX)
		.setPendingPrefix("")
		.setPendingSuffix("");

	OneInputStreamOperatorTestHarness<Tuple2<Integer, String>, Object> testHarness =
		createTestSink(sink, 1, 0);

	testHarness.setProcessingTime(0L);

	testHarness.setup();
	testHarness.open();

	for (int i = 0; i < numElements; i++) {
		testHarness.processElement(new StreamRecord<>(Tuple2.of(
			i, "message #" + Integer.toString(i)
		)));
	}

	testHarness.close();

	GenericData.setStringType(valueSchema, GenericData.StringType.String);
	Schema elementSchema = AvroKeyValueSinkWriter.AvroKeyValue.getSchema(keySchema, valueSchema);

	FSDataInputStream inStream = dfs.open(new Path(outPath + "/" + PART_PREFIX + "-0-0"));

	SpecificDatumReader<GenericRecord> elementReader = new SpecificDatumReader<>(elementSchema);
	DataFileStream<GenericRecord> dataFileStream = new DataFileStream<>(inStream, elementReader);
	for (int i = 0; i < numElements; i++) {
		AvroKeyValueSinkWriter.AvroKeyValue<Integer, String> wrappedEntry =
			new AvroKeyValueSinkWriter.AvroKeyValue<>(dataFileStream.next());
		int key = wrappedEntry.getKey();
		Assert.assertEquals(i, key);
		String value = wrappedEntry.getValue();
		Assert.assertEquals("message #" + i, value);
	}

	dataFileStream.close();
	inStream.close();
}
 

/**
 * Tests DeltaEvictor, evictAfter behavior.
 */
@Test
public void testDeltaEvictorEvictAfter() throws Exception {
	AtomicInteger closeCalled = new AtomicInteger(0);
	final int triggerCount = 2;
	final boolean evictAfter = true;
	final int threshold = 2;

	@SuppressWarnings({"unchecked", "rawtypes"})
	TypeSerializer<StreamRecord<Tuple2<String, Integer>>> streamRecordSerializer =
		(TypeSerializer<StreamRecord<Tuple2<String, Integer>>>) new StreamElementSerializer(STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));

	ListStateDescriptor<StreamRecord<Tuple2<String, Integer>>> stateDesc =
		new ListStateDescriptor<>("window-contents", streamRecordSerializer);

	EvictingWindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, GlobalWindow> operator = new EvictingWindowOperator<>(
		GlobalWindows.create(),
		new GlobalWindow.Serializer(),
		new TupleKeySelector(),
		BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
		stateDesc,
		new InternalIterableWindowFunction<>(new RichSumReducer<GlobalWindow>(closeCalled)),
		CountTrigger.of(triggerCount),
		DeltaEvictor.of(threshold, new DeltaFunction<Tuple2<String, Integer>>() {
			@Override
			public double getDelta(Tuple2<String, Integer> oldDataPoint, Tuple2<String, Integer> newDataPoint) {
				return newDataPoint.f1 - oldDataPoint.f1;
			}
		}, evictAfter),
		0,
		null /* late data output tag */);

	OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
		new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);

	long initialTime = 0L;
	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	testHarness.open();

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 3000));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 4), initialTime + 3999));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 20));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 5), initialTime + 999));

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 5), initialTime + 1998));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 6), initialTime + 1999));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1000));

	expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 5), Long.MAX_VALUE));
	expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 15), Long.MAX_VALUE));
	expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 2), Long.MAX_VALUE));

	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new ResultSortComparator());

	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 9), initialTime + 10999));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 10), initialTime + 1000));

	expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 16), Long.MAX_VALUE));
	expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 22), Long.MAX_VALUE));

	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new ResultSortComparator());

	testHarness.close();

	Assert.assertEquals("Close was not called.", 1, closeCalled.get());
}
 

protected void initializeState(OneInputStreamOperatorTestHarness testHarness) throws Exception{
	testHarness.setup();
	testHarness.initializeState(getOperatorSnapshotPath());
	testHarness.open();
}
 

@Test
public void testScalingDown() throws Exception {
	final File outDir = tempFolder.newFolder();

	OneInputStreamOperatorTestHarness<String, Object> testHarness1 = createRescalingTestSink(outDir, 3, 0, 100);
	testHarness1.setup();
	testHarness1.open();

	OneInputStreamOperatorTestHarness<String, Object> testHarness2 = createRescalingTestSink(outDir, 3, 1, 100);
	testHarness2.setup();
	testHarness2.open();

	OneInputStreamOperatorTestHarness<String, Object> testHarness3 = createRescalingTestSink(outDir, 3, 2, 100);
	testHarness3.setup();
	testHarness3.open();

	testHarness1.processElement(new StreamRecord<>("test1", 0L));
	checkLocalFs(outDir, 1, 0, 0, 0);

	testHarness2.processElement(new StreamRecord<>("test2", 0L));
	checkLocalFs(outDir, 2, 0, 0, 0);

	testHarness3.processElement(new StreamRecord<>("test3", 0L));
	testHarness3.processElement(new StreamRecord<>("test4", 0L));
	checkLocalFs(outDir, 4, 0, 0, 0);

	// intentionally we snapshot them in the reverse order so that the states are shuffled
	OperatorSubtaskState mergedSnapshot = AbstractStreamOperatorTestHarness.repackageState(
		testHarness3.snapshot(0, 0),
		testHarness1.snapshot(0, 0),
		testHarness2.snapshot(0, 0)
	);

	OperatorSubtaskState initState1 = AbstractStreamOperatorTestHarness.repartitionOperatorState(
		mergedSnapshot, maxParallelism, 3, 2, 0);

	testHarness1 = createRescalingTestSink(outDir, 2, 0, 100);
	testHarness1.setup();
	testHarness1.initializeState(initState1);
	testHarness1.open();

	checkLocalFs(outDir, 1, 0, 3, 3);

	OperatorSubtaskState initState2 = AbstractStreamOperatorTestHarness.repartitionOperatorState(
		mergedSnapshot, maxParallelism, 3, 2, 1);

	testHarness2 = createRescalingTestSink(outDir, 2, 1, 100);
	testHarness2.setup();
	testHarness2.initializeState(initState2);
	testHarness2.open();

	checkLocalFs(outDir, 0, 0, 4, 4);
}
 

@Test
public void testCheckpointBufferingWithMultipleBundles() throws Exception {
  FlinkPipelineOptions options = PipelineOptionsFactory.as(FlinkPipelineOptions.class);
  options.setMaxBundleSize(10L);
  options.setCheckpointingInterval(1L);

  TupleTag<String> outputTag = new TupleTag<>("main-output");

  StringUtf8Coder coder = StringUtf8Coder.of();
  WindowedValue.ValueOnlyWindowedValueCoder<String> windowedValueCoder =
      WindowedValue.getValueOnlyCoder(coder);

  DoFnOperator.MultiOutputOutputManagerFactory<String> outputManagerFactory =
      new DoFnOperator.MultiOutputOutputManagerFactory<>(
          outputTag,
          WindowedValue.getFullCoder(StringUtf8Coder.of(), GlobalWindow.Coder.INSTANCE));

  @SuppressWarnings("unchecked")
  Supplier<DoFnOperator<String, String>> doFnOperatorSupplier =
      () ->
          new DoFnOperator<>(
              new IdentityDoFn(),
              "stepName",
              windowedValueCoder,
              Collections.emptyMap(),
              outputTag,
              Collections.emptyList(),
              outputManagerFactory,
              WindowingStrategy.globalDefault(),
              new HashMap<>(), /* side-input mapping */
              Collections.emptyList(), /* side inputs */
              options,
              null,
              null,
              DoFnSchemaInformation.create(),
              Collections.emptyMap());

  DoFnOperator<String, String> doFnOperator = doFnOperatorSupplier.get();
  OneInputStreamOperatorTestHarness<WindowedValue<String>, WindowedValue<String>> testHarness =
      new OneInputStreamOperatorTestHarness<>(doFnOperator);

  testHarness.open();

  // start a bundle
  testHarness.processElement(
      new StreamRecord<>(WindowedValue.valueInGlobalWindow("regular element")));

  // This callback will be executed in the snapshotState function in the course of
  // finishing the currently active bundle. Everything emitted in the callback should
  // be buffered and not sent downstream.
  doFnOperator.setBundleFinishedCallback(
      () -> {
        try {
          // Clear this early for the test here because we want to finish the bundle from within
          // the callback which would otherwise cause an infinitive recursion
          doFnOperator.setBundleFinishedCallback(null);
          testHarness.processElement(
              new StreamRecord<>(WindowedValue.valueInGlobalWindow("trigger another bundle")));
          doFnOperator.invokeFinishBundle();
          testHarness.processElement(
              new StreamRecord<>(
                  WindowedValue.valueInGlobalWindow(
                      "check that the previous element is not flushed")));
        } catch (Exception e) {
          throw new RuntimeException(e);
        }
      });

  OperatorSubtaskState snapshot = testHarness.snapshot(0, 0);

  assertThat(
      stripStreamRecordFromWindowedValue(testHarness.getOutput()),
      contains(WindowedValue.valueInGlobalWindow("regular element")));
  testHarness.close();

  // Restore
  OneInputStreamOperatorTestHarness<WindowedValue<String>, WindowedValue<String>> testHarness2 =
      new OneInputStreamOperatorTestHarness<>(doFnOperatorSupplier.get());

  testHarness2.initializeState(snapshot);
  testHarness2.open();

  testHarness2.processElement(
      new StreamRecord<>(WindowedValue.valueInGlobalWindow("after restore")));

  assertThat(
      stripStreamRecordFromWindowedValue(testHarness2.getOutput()),
      contains(
          WindowedValue.valueInGlobalWindow("trigger another bundle"),
          WindowedValue.valueInGlobalWindow("check that the previous element is not flushed"),
          WindowedValue.valueInGlobalWindow("after restore")));
}
 

@Test
public void testReaderScalingDown() throws Exception {
	// simulates the scenario of scaling down from 2 to 1 instances

	final OneShotLatch waitingLatch = new OneShotLatch();

	// create the first instance and let it process the first split till element 5
	final OneShotLatch triggerLatch1 = new OneShotLatch();
	BlockingFileInputFormat format1 = new BlockingFileInputFormat(
		triggerLatch1, waitingLatch, new Path("test"), 20, 5);
	FileInputSplit[] splits = format1.createInputSplits(2);

	OneInputStreamOperatorTestHarness<TimestampedFileInputSplit, String> testHarness1 = getTestHarness(format1, 2, 0);
	testHarness1.open();
	testHarness1.processElement(new StreamRecord<>(getTimestampedSplit(0, splits[0])));

	// wait until its arrives to element 5
	if (!triggerLatch1.isTriggered()) {
		triggerLatch1.await();
	}

	// create the second instance and let it process the second split till element 15
	final OneShotLatch triggerLatch2 = new OneShotLatch();
	BlockingFileInputFormat format2 = new BlockingFileInputFormat(
		triggerLatch2, waitingLatch, new Path("test"), 20, 15);

	OneInputStreamOperatorTestHarness<TimestampedFileInputSplit, String> testHarness2 = getTestHarness(format2, 2, 1);
	testHarness2.open();
	testHarness2.processElement(new StreamRecord<>(getTimestampedSplit(0, splits[1])));

	// wait until its arrives to element 15
	if (!triggerLatch2.isTriggered()) {
		triggerLatch2.await();
	}

	// 1) clear the outputs of the two previous instances so that
	// we can compare their newly produced outputs with the merged one
	testHarness1.getOutput().clear();
	testHarness2.getOutput().clear();

	// 2) take the snapshots from the previous instances and merge them
	// into a new one which will be then used to initialize a third instance
	OperatorSubtaskState mergedState = AbstractStreamOperatorTestHarness.
		repackageState(
			testHarness2.snapshot(0, 0),
			testHarness1.snapshot(0, 0)
		);

	// 3) and repartition to get the initialized state when scaling down.
	OperatorSubtaskState initState =
		AbstractStreamOperatorTestHarness.repartitionOperatorState(mergedState, maxParallelism, 2, 1, 0);

	// create the third instance
	final OneShotLatch wLatch = new OneShotLatch();
	final OneShotLatch tLatch = new OneShotLatch();

	BlockingFileInputFormat format = new BlockingFileInputFormat(wLatch, tLatch, new Path("test"), 20, 5);
	OneInputStreamOperatorTestHarness<TimestampedFileInputSplit, String> testHarness = getTestHarness(format, 1, 0);

	// initialize the state of the new operator with the constructed by
	// combining the partial states of the instances above.
	testHarness.initializeState(initState);
	testHarness.open();

	// now restart the waiting operators
	wLatch.trigger();
	tLatch.trigger();
	waitingLatch.trigger();

	// and wait for the processing to finish
	synchronized (testHarness1.getCheckpointLock()) {
		testHarness1.close();
	}
	synchronized (testHarness2.getCheckpointLock()) {
		testHarness2.close();
	}
	synchronized (testHarness.getCheckpointLock()) {
		testHarness.close();
	}

	Queue<Object> expectedResult = new ArrayDeque<>();
	putElementsInQ(expectedResult, testHarness1.getOutput());
	putElementsInQ(expectedResult, testHarness2.getOutput());

	Queue<Object> actualResult = new ArrayDeque<>();
	putElementsInQ(actualResult, testHarness.getOutput());

	Assert.assertEquals(20, actualResult.size());
	Assert.assertArrayEquals(expectedResult.toArray(), actualResult.toArray());
}
 

@Test
public void testRestoreApplyProcessingTimeWindows() throws Exception {
	final int windowSize = 3;

	ListStateDescriptor<Tuple2<String, Integer>> stateDesc = new ListStateDescriptor<>("window-contents",
			STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));

	WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>(
			TumblingProcessingTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)),
			new TimeWindow.Serializer(),
			new TupleKeySelector<>(),
			BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
			stateDesc,
			new InternalIterableWindowFunction<>(new RichSumReducer<TimeWindow>()),
			ProcessingTimeTrigger.create(),
			0,
			null /* late data output tag */);

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector<>(), BasicTypeInfo.STRING_TYPE_INFO);

	testHarness.setup();

	testHarness.initializeState(
		OperatorSnapshotUtil.getResourceFilename(
			"win-op-migration-test-apply-processing-time-flink" + testMigrateVersion + "-snapshot"));

	testHarness.open();

	testHarness.setProcessingTime(3020);
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 3)));
	testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 3)));

	testHarness.setProcessingTime(6000);

	expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 3), 5999));
	expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 4), 5999));
	expectedOutput.add(new StreamRecord<>(new Tuple2<>("key3", 1), 5999));

	TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator<>());
	testHarness.close();
}
 

@Test
public void testLateDroppingForStatefulFn() throws Exception {

  WindowingStrategy<Object, IntervalWindow> windowingStrategy =
      WindowingStrategy.of(FixedWindows.of(new Duration(10)));

  DoFn<Integer, String> fn =
      new DoFn<Integer, String>() {

        @StateId("state")
        private final StateSpec<ValueState<String>> stateSpec =
            StateSpecs.value(StringUtf8Coder.of());

        @ProcessElement
        public void processElement(ProcessContext context) {
          context.output(context.element().toString());
        }
      };

  VarIntCoder keyCoder = VarIntCoder.of();
  Coder<WindowedValue<Integer>> inputCoder =
      WindowedValue.getFullCoder(keyCoder, windowingStrategy.getWindowFn().windowCoder());
  Coder<WindowedValue<String>> outputCoder =
      WindowedValue.getFullCoder(
          StringUtf8Coder.of(), windowingStrategy.getWindowFn().windowCoder());

  KeySelector<WindowedValue<Integer>, ByteBuffer> keySelector =
      e -> FlinkKeyUtils.encodeKey(e.getValue(), keyCoder);

  TupleTag<String> outputTag = new TupleTag<>("main-output");

  DoFnOperator<Integer, String> doFnOperator =
      new DoFnOperator<>(
          fn,
          "stepName",
          inputCoder,
          Collections.emptyMap(),
          outputTag,
          Collections.emptyList(),
          new DoFnOperator.MultiOutputOutputManagerFactory<>(outputTag, outputCoder),
          windowingStrategy,
          new HashMap<>(), /* side-input mapping */
          Collections.emptyList(), /* side inputs */
          PipelineOptionsFactory.as(FlinkPipelineOptions.class),
          keyCoder, /* key coder */
          keySelector,
          DoFnSchemaInformation.create(),
          Collections.emptyMap());

  OneInputStreamOperatorTestHarness<WindowedValue<Integer>, WindowedValue<String>> testHarness =
      new KeyedOneInputStreamOperatorTestHarness<>(
          doFnOperator,
          keySelector,
          new CoderTypeInformation<>(FlinkKeyUtils.ByteBufferCoder.of()));

  testHarness.open();

  testHarness.processWatermark(0);

  IntervalWindow window1 = new IntervalWindow(new Instant(0), Duration.millis(10));

  // this should not be late
  testHarness.processElement(
      new StreamRecord<>(WindowedValue.of(13, new Instant(0), window1, PaneInfo.NO_FIRING)));

  assertThat(
      stripStreamRecordFromWindowedValue(testHarness.getOutput()),
      contains(WindowedValue.of("13", new Instant(0), window1, PaneInfo.NO_FIRING)));

  testHarness.getOutput().clear();

  testHarness.processWatermark(9);

  // this should still not be considered late
  testHarness.processElement(
      new StreamRecord<>(WindowedValue.of(17, new Instant(0), window1, PaneInfo.NO_FIRING)));

  assertThat(
      stripStreamRecordFromWindowedValue(testHarness.getOutput()),
      contains(WindowedValue.of("17", new Instant(0), window1, PaneInfo.NO_FIRING)));

  testHarness.getOutput().clear();

  testHarness.processWatermark(10);

  // this should now be considered late
  testHarness.processElement(
      new StreamRecord<>(WindowedValue.of(17, new Instant(0), window1, PaneInfo.NO_FIRING)));

  assertThat(stripStreamRecordFromWindowedValue(testHarness.getOutput()), emptyIterable());

  testHarness.close();
}