org.apache.flink.streaming.api.functions.source.TimestampedFileInputSplit Java Examples

@Test
public void testSplitEquality() {

	TimestampedFileInputSplit richFirstSplit =
		new TimestampedFileInputSplit(10, 2, new Path("test"), 0, 100, null);

	TimestampedFileInputSplit richSecondSplit =
		new TimestampedFileInputSplit(10, 2, new Path("test"), 0, 100, null);
	Assert.assertEquals(richFirstSplit, richSecondSplit);

	TimestampedFileInputSplit richModSecondSplit =
		new TimestampedFileInputSplit(11, 2, new Path("test"), 0, 100, null);
	Assert.assertNotEquals(richSecondSplit, richModSecondSplit);

	TimestampedFileInputSplit richThirdSplit =
		new TimestampedFileInputSplit(10, 2, new Path("test/test1"), 0, 100, null);
	Assert.assertEquals(richThirdSplit.getModificationTime(), 10);
	Assert.assertNotEquals(richFirstSplit, richThirdSplit);

	TimestampedFileInputSplit richThirdSplitCopy =
		new TimestampedFileInputSplit(10, 2, new Path("test/test1"), 0, 100, null);
	Assert.assertEquals(richThirdSplitCopy, richThirdSplit);
}
 

@Test
public void testInvalidPathSpecification() throws Exception {

	String invalidPath = "hdfs://" + hdfsCluster.getURI().getHost() + ":" + hdfsCluster.getNameNodePort() + "/invalid/";
	TextInputFormat format = new TextInputFormat(new Path(invalidPath));

	ContinuousFileMonitoringFunction<String> monitoringFunction =
		new ContinuousFileMonitoringFunction<>(format,
			FileProcessingMode.PROCESS_ONCE, 1, INTERVAL);
	try {
		monitoringFunction.run(new DummySourceContext() {
			@Override
			public void collect(TimestampedFileInputSplit element) {
				// we should never arrive here with an invalid path
				Assert.fail("Test passes with an invalid path.");
			}
		});

		// we should never arrive here with an invalid path
		Assert.fail("Test passed with an invalid path.");

	} catch (FileNotFoundException e) {
		Assert.assertEquals("The provided file path " + format.getFilePath() + " does not exist.", e.getMessage());
	}
}
 

@Test
public void testSplitEquality() {

	TimestampedFileInputSplit richFirstSplit =
		new TimestampedFileInputSplit(10, 2, new Path("test"), 0, 100, null);

	TimestampedFileInputSplit richSecondSplit =
		new TimestampedFileInputSplit(10, 2, new Path("test"), 0, 100, null);
	Assert.assertEquals(richFirstSplit, richSecondSplit);

	TimestampedFileInputSplit richModSecondSplit =
		new TimestampedFileInputSplit(11, 2, new Path("test"), 0, 100, null);
	Assert.assertNotEquals(richSecondSplit, richModSecondSplit);

	TimestampedFileInputSplit richThirdSplit =
		new TimestampedFileInputSplit(10, 2, new Path("test/test1"), 0, 100, null);
	Assert.assertEquals(richThirdSplit.getModificationTime(), 10);
	Assert.assertNotEquals(richFirstSplit, richThirdSplit);

	TimestampedFileInputSplit richThirdSplitCopy =
		new TimestampedFileInputSplit(10, 2, new Path("test/test1"), 0, 100, null);
	Assert.assertEquals(richThirdSplitCopy, richThirdSplit);
}
 

@Test
public void testSplitEquality() {

	TimestampedFileInputSplit richFirstSplit =
		new TimestampedFileInputSplit(10, 2, new Path("test"), 0, 100, null);

	TimestampedFileInputSplit richSecondSplit =
		new TimestampedFileInputSplit(10, 2, new Path("test"), 0, 100, null);
	Assert.assertEquals(richFirstSplit, richSecondSplit);

	TimestampedFileInputSplit richModSecondSplit =
		new TimestampedFileInputSplit(11, 2, new Path("test"), 0, 100, null);
	Assert.assertNotEquals(richSecondSplit, richModSecondSplit);

	TimestampedFileInputSplit richThirdSplit =
		new TimestampedFileInputSplit(10, 2, new Path("test/test1"), 0, 100, null);
	Assert.assertEquals(richThirdSplit.getModificationTime(), 10);
	Assert.assertNotEquals(richFirstSplit, richThirdSplit);

	TimestampedFileInputSplit richThirdSplitCopy =
		new TimestampedFileInputSplit(10, 2, new Path("test/test1"), 0, 100, null);
	Assert.assertEquals(richThirdSplitCopy, richThirdSplit);
}
 

@Test
public void testInvalidPathSpecification() throws Exception {

	String invalidPath = "hdfs://" + hdfsCluster.getURI().getHost() + ":" + hdfsCluster.getNameNodePort() + "/invalid/";
	TextInputFormat format = new TextInputFormat(new Path(invalidPath));

	ContinuousFileMonitoringFunction<String> monitoringFunction =
		new ContinuousFileMonitoringFunction<>(format,
			FileProcessingMode.PROCESS_ONCE, 1, INTERVAL);
	try {
		monitoringFunction.run(new DummySourceContext() {
			@Override
			public void collect(TimestampedFileInputSplit element) {
				// we should never arrive here with an invalid path
				Assert.fail("Test passes with an invalid path.");
			}
		});

		// we should never arrive here with an invalid path
		Assert.fail("Test passed with an invalid path.");

	} catch (FileNotFoundException e) {
		Assert.assertEquals("The provided file path " + format.getFilePath() + " does not exist.", e.getMessage());
	}
}
 

@Test
public void testInvalidPathSpecification() throws Exception {

	String invalidPath = "hdfs://" + hdfsCluster.getURI().getHost() + ":" + hdfsCluster.getNameNodePort() + "/invalid/";
	TextInputFormat format = new TextInputFormat(new Path(invalidPath));

	ContinuousFileMonitoringFunction<String> monitoringFunction =
		new ContinuousFileMonitoringFunction<>(format,
			FileProcessingMode.PROCESS_ONCE, 1, INTERVAL);
	try {
		monitoringFunction.run(new DummySourceContext() {
			@Override
			public void collect(TimestampedFileInputSplit element) {
				// we should never arrive here with an invalid path
				Assert.fail("Test passes with an invalid path.");
			}
		});

		// we should never arrive here with an invalid path
		Assert.fail("Test passed with an invalid path.");

	} catch (FileNotFoundException e) {
		Assert.assertEquals("The provided file path " + format.getFilePath() + " does not exist.", e.getMessage());
	}
}
 

private HarnessWithFormat buildAndStart(
		int noOfTasks,
		int taskIdx,
		int elementsBeforeCheckpoint,
		@Nullable OperatorSubtaskState initState,
		FileInputSplit... splits) throws Exception {

	BlockingFileInputFormat format = new BlockingFileInputFormat(new Path("test"), sizeOfSplit, elementsBeforeCheckpoint);

	OneInputStreamOperatorTestHarness<TimestampedFileInputSplit, String> harness = getTestHarness(format, noOfTasks, taskIdx);
	harness.setup();
	if (initState != null) {
		harness.initializeState(initState);
	}
	harness.open();
	if (splits != null) {
		for (int i = 0; i < splits.length; i++) {
			harness.processElement(new StreamRecord<>(getTimestampedSplit(i, splits[i])));
		}
	}
	HarnessWithFormat harnessWithFormat = new HarnessWithFormat(harness, format);
	while (!format.isFirstChunkProcessed()) {
		harnessWithFormat.mailboxProcessor.runMailboxStep();
	}
	return harnessWithFormat;
}
 

private OneInputStreamOperatorTestHarness<TimestampedFileInputSplit, String> getTestHarness(
	BlockingFileInputFormat format, int noOfTasks, int taksIdx) throws Exception {

	ContinuousFileReaderOperator<String> reader = new ContinuousFileReaderOperator<>(format);
	reader.setOutputType(TypeExtractor.getInputFormatTypes(format), new ExecutionConfig());

	OneInputStreamOperatorTestHarness<TimestampedFileInputSplit, String> testHarness =
		new OneInputStreamOperatorTestHarness<>(reader, maxParallelism, noOfTasks, taksIdx);
	testHarness.setTimeCharacteristic(TimeCharacteristic.EventTime);
	return testHarness;
}
 

private TimestampedFileInputSplit getTimestampedSplit(long modTime, FileInputSplit split) {
	Preconditions.checkNotNull(split);
	return new TimestampedFileInputSplit(
		modTime,
		split.getSplitNumber(),
		split.getPath(),
		split.getStart(),
		split.getLength(),
		split.getHostnames());
}
 

private FileInputSplit createSplitFromTimestampedSplit(TimestampedFileInputSplit split) {
	Preconditions.checkNotNull(split);

	return new FileInputSplit(
		split.getSplitNumber(),
		split.getPath(),
		split.getStart(),
		split.getLength(),
		split.getHostnames()
	);
}
 

@Test
public void testSplitComparison() {
	TimestampedFileInputSplit richFirstSplit =
		new TimestampedFileInputSplit(0, 3, new Path("test/test1"), 0, 100, null);

	TimestampedFileInputSplit richSecondSplit =
		new TimestampedFileInputSplit(10, 2, new Path("test/test2"), 0, 100, null);

	TimestampedFileInputSplit richThirdSplit =
		new TimestampedFileInputSplit(10, 1, new Path("test/test2"), 0, 100, null);

	TimestampedFileInputSplit richForthSplit =
		new TimestampedFileInputSplit(11, 0, new Path("test/test3"), 0, 100, null);

	TimestampedFileInputSplit richFifthSplit =
		new TimestampedFileInputSplit(11, 1, new Path("test/test3"), 0, 100, null);

	// smaller mod time
	Assert.assertTrue(richFirstSplit.compareTo(richSecondSplit) < 0);

	// lexicographically on the path
	Assert.assertTrue(richThirdSplit.compareTo(richFifthSplit) < 0);

	// same mod time, same file so smaller split number first
	Assert.assertTrue(richThirdSplit.compareTo(richSecondSplit) < 0);

	// smaller modification time first
	Assert.assertTrue(richThirdSplit.compareTo(richForthSplit) < 0);
}
 

@Test
public void testIllegalArgument() {
	try {
		new TimestampedFileInputSplit(-10, 2, new Path("test"), 0, 100, null); // invalid modification time
	} catch (Exception e) {
		if (!(e instanceof IllegalArgumentException)) {
			Assert.fail(e.getMessage());
		}
	}
}
 

private OneInputStreamOperatorTestHarness<TimestampedFileInputSplit, String> getTestHarness(
	BlockingFileInputFormat format, int noOfTasks, int taksIdx) throws Exception {

	ContinuousFileReaderOperator<String> reader = new ContinuousFileReaderOperator<>(format);
	reader.setOutputType(TypeExtractor.getInputFormatTypes(format), new ExecutionConfig());

	OneInputStreamOperatorTestHarness<TimestampedFileInputSplit, String> testHarness =
		new OneInputStreamOperatorTestHarness<>(reader, maxParallelism, noOfTasks, taksIdx);
	testHarness.setTimeCharacteristic(TimeCharacteristic.EventTime);
	return testHarness;
}
 

private FileInputSplit createSplitFromTimestampedSplit(TimestampedFileInputSplit split) {
	checkNotNull(split);

	return new FileInputSplit(
		split.getSplitNumber(),
		split.getPath(),
		split.getStart(),
		split.getLength(),
		split.getHostnames()
	);
}
 

@Test
public void testSplitComparison() {
	TimestampedFileInputSplit richFirstSplit =
		new TimestampedFileInputSplit(0, 3, new Path("test/test1"), 0, 100, null);

	TimestampedFileInputSplit richSecondSplit =
		new TimestampedFileInputSplit(10, 2, new Path("test/test2"), 0, 100, null);

	TimestampedFileInputSplit richThirdSplit =
		new TimestampedFileInputSplit(10, 1, new Path("test/test2"), 0, 100, null);

	TimestampedFileInputSplit richForthSplit =
		new TimestampedFileInputSplit(11, 0, new Path("test/test3"), 0, 100, null);

	TimestampedFileInputSplit richFifthSplit =
		new TimestampedFileInputSplit(11, 1, new Path("test/test3"), 0, 100, null);

	// smaller mod time
	Assert.assertTrue(richFirstSplit.compareTo(richSecondSplit) < 0);

	// lexicographically on the path
	Assert.assertTrue(richThirdSplit.compareTo(richFifthSplit) < 0);

	// same mod time, same file so smaller split number first
	Assert.assertTrue(richThirdSplit.compareTo(richSecondSplit) < 0);

	// smaller modification time first
	Assert.assertTrue(richThirdSplit.compareTo(richForthSplit) < 0);
}
 

private TimestampedFileInputSplit getTimestampedSplit(long modTime, FileInputSplit split) {
	Preconditions.checkNotNull(split);
	return new TimestampedFileInputSplit(
		modTime,
		split.getSplitNumber(),
		split.getPath(),
		split.getStart(),
		split.getLength(),
		split.getHostnames());
}
 

private FileInputSplit createSplitFromTimestampedSplit(TimestampedFileInputSplit split) {
	Preconditions.checkNotNull(split);

	return new FileInputSplit(
		split.getSplitNumber(),
		split.getPath(),
		split.getStart(),
		split.getLength(),
		split.getHostnames()
	);
}
 

@Override
public void collect(TimestampedFileInputSplit element) {
	String seenFileName = element.getPath().getName();
	this.seenFiles.add(seenFileName);

	if (seenFiles.size() == elementsBeforeNotifying && !latch.isTriggered()) {
		latch.trigger();
	}

	if (seenFiles.size() == elementsBeforeCanceling) {
		src.cancel();
	}
}
 

@Override
public void collect(TimestampedFileInputSplit element) {
	try {
		long modTime = hdfs.getFileStatus(new org.apache.hadoop.fs.Path(element.getPath().getPath())).getModificationTime();

		Assert.assertTrue(modTime >= lastSeenModTime);
		Assert.assertEquals(expectedModificationTimes[splitCounter], modTime);

		lastSeenModTime = modTime;
		splitCounter++;
	} catch (IOException e) {
		Assert.fail(e.getMessage());
	}
}
 

public static <OUT> OneInputStreamOperatorTestHarness<TimestampedFileInputSplit, OUT> createContinuousFileProcessingTestHarness(
	FileInputFormat<OUT> inputFormat,
	TypeInformation<OUT> outTypeInfo,
	ExecutionConfig executionConfig) throws Exception {

	OneInputStreamOperatorTestHarness<TimestampedFileInputSplit, OUT> testHarness =
		new OneInputStreamOperatorTestHarness<>(new ContinuousFileReaderOperatorFactory<>(inputFormat));
	testHarness.getOperatorFactory().setOutputType(
		outTypeInfo,
		executionConfig == null ? testHarness.getExecutionConfig() : executionConfig);

	return testHarness;
}
 

private TimestampedFileInputSplit getTimestampedSplit(long modTime, FileInputSplit split) {
	Preconditions.checkNotNull(split);
	return new TimestampedFileInputSplit(
		modTime,
		split.getSplitNumber(),
		split.getPath(),
		split.getStart(),
		split.getLength(),
		split.getHostnames());
}
 

private DataStream<RowData> createStreamSourceForNonPartitionTable(
		StreamExecutionEnvironment execEnv,
		TypeInformation<RowData> typeInfo,
		HiveTableInputFormat inputFormat,
		HiveTablePartition hiveTable) {
	HiveTableFileInputFormat fileInputFormat = new HiveTableFileInputFormat(inputFormat, hiveTable);

	Configuration configuration = new Configuration();
	catalogTable.getOptions().forEach(configuration::setString);
	String consumeOrderStr = configuration.get(STREAMING_SOURCE_CONSUME_ORDER);
	ConsumeOrder consumeOrder = ConsumeOrder.getConsumeOrder(consumeOrderStr);
	if (consumeOrder != ConsumeOrder.CREATE_TIME_ORDER) {
		throw new UnsupportedOperationException(
				"Only " + ConsumeOrder.CREATE_TIME_ORDER + " is supported for non partition table.");
	}

	String consumeOffset = configuration.get(STREAMING_SOURCE_CONSUME_START_OFFSET);
	// to Local zone mills instead of UTC mills
	long currentReadTime = TimestampData.fromLocalDateTime(toLocalDateTime(consumeOffset))
			.toTimestamp().getTime();

	Duration monitorInterval = configuration.get(STREAMING_SOURCE_MONITOR_INTERVAL);

	ContinuousFileMonitoringFunction<RowData> monitoringFunction =
			new ContinuousFileMonitoringFunction<>(
					fileInputFormat,
					FileProcessingMode.PROCESS_CONTINUOUSLY,
					execEnv.getParallelism(),
					monitorInterval.toMillis(),
					currentReadTime);

	ContinuousFileReaderOperatorFactory<RowData, TimestampedFileInputSplit> factory =
			new ContinuousFileReaderOperatorFactory<>(fileInputFormat);

	String sourceName = "HiveFileMonitoringFunction";
	SingleOutputStreamOperator<RowData> source = execEnv.addSource(monitoringFunction, sourceName)
			.transform("Split Reader: " + sourceName, typeInfo, factory);

	return new DataStreamSource<>(source);
}
 

private OneInputStreamOperatorTestHarness<TimestampedFileInputSplit, String> getTestHarness(
	BlockingFileInputFormat format,
	int noOfTasks,
	int taskIdx) throws Exception {
	OneInputStreamOperatorTestHarness<TimestampedFileInputSplit, String> testHarness =
		new OneInputStreamOperatorTestHarness<>(
			new ContinuousFileReaderOperatorFactory<>(format, TypeExtractor.getInputFormatTypes(format), new ExecutionConfig()),
			maxParallelism,
			noOfTasks,
			taskIdx);
	testHarness.setTimeCharacteristic(TimeCharacteristic.EventTime);
	return testHarness;
}
 

@Test
public void testSplitComparison() {
	TimestampedFileInputSplit richFirstSplit =
		new TimestampedFileInputSplit(0, 3, new Path("test/test1"), 0, 100, null);

	TimestampedFileInputSplit richSecondSplit =
		new TimestampedFileInputSplit(10, 2, new Path("test/test2"), 0, 100, null);

	TimestampedFileInputSplit richThirdSplit =
		new TimestampedFileInputSplit(10, 1, new Path("test/test2"), 0, 100, null);

	TimestampedFileInputSplit richForthSplit =
		new TimestampedFileInputSplit(11, 0, new Path("test/test3"), 0, 100, null);

	TimestampedFileInputSplit richFifthSplit =
		new TimestampedFileInputSplit(11, 1, new Path("test/test3"), 0, 100, null);

	// smaller mod time
	Assert.assertTrue(richFirstSplit.compareTo(richSecondSplit) < 0);

	// lexicographically on the path
	Assert.assertTrue(richThirdSplit.compareTo(richFifthSplit) < 0);

	// same mod time, same file so smaller split number first
	Assert.assertTrue(richThirdSplit.compareTo(richSecondSplit) < 0);

	// smaller modification time first
	Assert.assertTrue(richThirdSplit.compareTo(richForthSplit) < 0);
}
 

@Test
public void testIllegalArgument() {
	try {
		new TimestampedFileInputSplit(-10, 2, new Path("test"), 0, 100, null); // invalid modification time
	} catch (Exception e) {
		if (!(e instanceof IllegalArgumentException)) {
			Assert.fail(e.getMessage());
		}
	}
}
 

private FileInputSplit createSplitFromTimestampedSplit(TimestampedFileInputSplit split) {
	checkNotNull(split);

	return new FileInputSplit(
		split.getSplitNumber(),
		split.getPath(),
		split.getStart(),
		split.getLength(),
		split.getHostnames()
	);
}
 

private <OUT> DataStreamSource<OUT> createFileInput(FileInputFormat<OUT> inputFormat,
													TypeInformation<OUT> typeInfo,
													String sourceName,
													FileProcessingMode monitoringMode,
													long interval) {

	Preconditions.checkNotNull(inputFormat, "Unspecified file input format.");
	Preconditions.checkNotNull(typeInfo, "Unspecified output type information.");
	Preconditions.checkNotNull(sourceName, "Unspecified name for the source.");
	Preconditions.checkNotNull(monitoringMode, "Unspecified monitoring mode.");

	Preconditions.checkArgument(monitoringMode.equals(FileProcessingMode.PROCESS_ONCE) ||
			interval >= ContinuousFileMonitoringFunction.MIN_MONITORING_INTERVAL,
		"The path monitoring interval cannot be less than " +
				ContinuousFileMonitoringFunction.MIN_MONITORING_INTERVAL + " ms.");

	ContinuousFileMonitoringFunction<OUT> monitoringFunction =
		new ContinuousFileMonitoringFunction<>(inputFormat, monitoringMode, getParallelism(), interval);

	ContinuousFileReaderOperatorFactory<OUT, TimestampedFileInputSplit> factory =
			new ContinuousFileReaderOperatorFactory<>(inputFormat);

	SingleOutputStreamOperator<OUT> source = addSource(monitoringFunction, sourceName)
			.transform("Split Reader: " + sourceName, typeInfo, factory);

	return new DataStreamSource<>(source);
}
 

private FileInputSplit createSplitFromTimestampedSplit(TimestampedFileInputSplit split) {
	Preconditions.checkNotNull(split);

	return new FileInputSplit(
		split.getSplitNumber(),
		split.getPath(),
		split.getStart(),
		split.getLength(),
		split.getHostnames()
	);
}
 

@Override
public void collect(TimestampedFileInputSplit element) {
	String seenFileName = element.getPath().getName();
	this.seenFiles.add(seenFileName);

	if (seenFiles.size() == elementsBeforeNotifying && !latch.isTriggered()) {
		latch.trigger();
	}

	if (seenFiles.size() == elementsBeforeCanceling) {
		src.cancel();
	}
}
 

@Override
public void collect(TimestampedFileInputSplit element) {
	try {
		long modTime = hdfs.getFileStatus(new org.apache.hadoop.fs.Path(element.getPath().getPath())).getModificationTime();

		Assert.assertTrue(modTime >= lastSeenModTime);
		Assert.assertEquals(expectedModificationTimes[splitCounter], modTime);

		lastSeenModTime = modTime;
		splitCounter++;
	} catch (IOException e) {
		Assert.fail(e.getMessage());
	}
}