org.apache.flink.testutils.TestFileUtils Java Examples

The following examples show how to use org.apache.flink.testutils.TestFileUtils. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: FileInputFormatTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testGetStatisticsOneFileNoCachedVersion() {
	try {
		final long SIZE = 1024 * 500;
		String tempFile = TestFileUtils.createTempFile(SIZE);
		
		final DummyFileInputFormat format = new DummyFileInputFormat();
		format.setFilePath(tempFile);
		format.configure(new Configuration());
		
		BaseStatistics stats = format.getStatistics(null);
		Assert.assertEquals("The file size from the statistics is wrong.", SIZE, stats.getTotalInputSize());
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
Example #2
Source File: DelimitedInputFormatSamplingTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testSamplingOverlyLongRecord() {
	try {
		final String tempFile = TestFileUtils.createTempFile(2 * OptimizerOptions.DELIMITED_FORMAT_MAX_SAMPLE_LEN.defaultValue());
		final Configuration conf = new Configuration();
		
		final TestDelimitedInputFormat format = new TestDelimitedInputFormat(CONFIG);
		format.setFilePath(tempFile);
		format.configure(conf);
		
		Assert.assertNull("Expected exception due to overly long record.", format.getStatistics(null));
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
Example #3
Source File: EnumerateNestedFilesTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
/**
 * Test without nested directory and recursive.file.enumeration = true
 */
@Test
public void testNoNestedDirectoryTrue() {
	try {
		String filePath = TestFileUtils.createTempFile("foo");

		this.format.setFilePath(new Path(filePath));
		this.config.setBoolean("recursive.file.enumeration", true);
		format.configure(this.config);

		FileInputSplit[] splits = format.createInputSplits(1);
		Assert.assertEquals(1, splits.length);
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
Example #4
Source File: EnumerateNestedFilesTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Test without nested directory and recursive.file.enumeration = true
 */
@Test
public void testNoNestedDirectoryTrue() {
	try {
		String filePath = TestFileUtils.createTempFile("foo");

		this.format.setFilePath(new Path(filePath));
		this.config.setBoolean("recursive.file.enumeration", true);
		format.configure(this.config);

		FileInputSplit[] splits = format.createInputSplits(1);
		Assert.assertEquals(1, splits.length);
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
Example #5
Source File: DelimitedInputFormatSamplingTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testSamplingOneFile() {
	try {
		final String tempFile = TestFileUtils.createTempFile(TEST_DATA1);
		final Configuration conf = new Configuration();
		
		final TestDelimitedInputFormat format = new TestDelimitedInputFormat(CONFIG);
		format.setFilePath(tempFile);
		format.configure(conf);
		BaseStatistics stats = format.getStatistics(null);
		
		final int numLines = TEST_DATA_1_LINES;
		final float avgWidth = ((float) TEST_DATA1.length()) / TEST_DATA_1_LINES;
		Assert.assertTrue("Wrong record count.", stats.getNumberOfRecords() < numLines + 1 & stats.getNumberOfRecords() > numLines - 1);
		Assert.assertTrue("Wrong avg record size.", stats.getAverageRecordWidth() < avgWidth + 1 & stats.getAverageRecordWidth() > avgWidth - 1);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
Example #6
Source File: EnumerateNestedFilesTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Test with one nested directory and recursive.file.enumeration = true
 */
@Test
public void testOneNestedDirectoryTrue() {
	try {
		String firstLevelDir = TestFileUtils.randomFileName();
		String secondLevelDir = TestFileUtils.randomFileName();

		File insideNestedDir = tempFolder.newFolder(firstLevelDir, secondLevelDir);
		File nestedDir = insideNestedDir.getParentFile();

		// create a file in the first-level and two files in the nested dir
		TestFileUtils.createTempFileInDirectory(nestedDir.getAbsolutePath(), "paella");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "kalamari");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "fideua");

		this.format.setFilePath(new Path(nestedDir.toURI().toString()));
		this.config.setBoolean("recursive.file.enumeration", true);
		format.configure(this.config);

		FileInputSplit[] splits = format.createInputSplits(1);
		Assert.assertEquals(3, splits.length);
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
Example #7
Source File: EnumerateNestedFilesTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Test with one nested directory and recursive.file.enumeration = false
 */
@Test
public void testOneNestedDirectoryFalse() {
	try {
		String firstLevelDir = TestFileUtils.randomFileName();
		String secondLevelDir = TestFileUtils.randomFileName();

		File insideNestedDir = tempFolder.newFolder(firstLevelDir, secondLevelDir);
		File nestedDir = insideNestedDir.getParentFile();

		// create a file in the first-level and two files in the nested dir
		TestFileUtils.createTempFileInDirectory(nestedDir.getAbsolutePath(), "paella");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "kalamari");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "fideua");

		this.format.setFilePath(new Path(nestedDir.toURI().toString()));
		this.config.setBoolean("recursive.file.enumeration", false);
		format.configure(this.config);

		FileInputSplit[] splits = format.createInputSplits(1);
		Assert.assertEquals(1, splits.length);
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
Example #8
Source File: FileInputFormatTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testGetStatisticsMultipleFilesMultiplePathsNoCachedVersion() throws IOException {
	final long size1 = 2077;
	final long size2 = 31909;
	final long size3 = 10;
	final long totalSize123 = size1 + size2 + size3;
	
	String tempDir = TestFileUtils.createTempFileDir(temporaryFolder.newFolder(), size1, size2, size3);
	
	final long size4 = 2051;
	final long size5 = 31902;
	final long size6 = 15;
	final long totalSize456 = size4 + size5 + size6;
	String tempDir2 = TestFileUtils.createTempFileDir(temporaryFolder.newFolder(), size4, size5, size6);

	final MultiDummyFileInputFormat format = new MultiDummyFileInputFormat();
	format.setFilePaths(tempDir, tempDir2);
	format.configure(new Configuration());
	
	BaseStatistics stats = format.getStatistics(null);
	Assert.assertEquals("The file size from the statistics is wrong.", totalSize123 + totalSize456, stats.getTotalInputSize());
}
Example #9
Source File: FileInputFormatTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testGetStatisticsMultipleOneFileNoCachedVersion() throws IOException {
	final long size1 = 1024 * 500;
	String tempFile = TestFileUtils.createTempFile(size1);

	final long size2 = 1024 * 505;
	String tempFile2 = TestFileUtils.createTempFile(size2);

	final long totalSize = size1 + size2;
	
	final MultiDummyFileInputFormat format = new MultiDummyFileInputFormat();
	format.setFilePaths(tempFile, tempFile2);
	format.configure(new Configuration());
	
	BaseStatistics stats = format.getStatistics(null);
	Assert.assertEquals("The file size from the statistics is wrong.", totalSize, stats.getTotalInputSize());
}
Example #10
Source File: FileInputFormatTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testGetStatisticsMultipleFilesNoCachedVersion() {
	try {
		final long SIZE1 = 2077;
		final long SIZE2 = 31909;
		final long SIZE3 = 10;
		final long TOTAL = SIZE1 + SIZE2 + SIZE3;
		
		String tempDir = TestFileUtils.createTempFileDir(temporaryFolder.newFolder(), SIZE1, SIZE2, SIZE3);
		
		final DummyFileInputFormat format = new DummyFileInputFormat();
		format.setFilePath(tempDir);
		format.configure(new Configuration());
		
		BaseStatistics stats = format.getStatistics(null);
		Assert.assertEquals("The file size from the statistics is wrong.", TOTAL, stats.getTotalInputSize());
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
Example #11
Source File: FileInputFormatTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testGetStatisticsOneFileNoCachedVersion() {
	try {
		final long SIZE = 1024 * 500;
		String tempFile = TestFileUtils.createTempFile(SIZE);
		
		final DummyFileInputFormat format = new DummyFileInputFormat();
		format.setFilePath(tempFile);
		format.configure(new Configuration());
		
		BaseStatistics stats = format.getStatistics(null);
		Assert.assertEquals("The file size from the statistics is wrong.", SIZE, stats.getTotalInputSize());
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
Example #12
Source File: EnumerateNestedFilesTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
/**
 * Test with one nested directory and recursive.file.enumeration = false
 */
@Test
public void testOneNestedDirectoryFalse() {
	try {
		String firstLevelDir = TestFileUtils.randomFileName();
		String secondLevelDir = TestFileUtils.randomFileName();

		File insideNestedDir = tempFolder.newFolder(firstLevelDir, secondLevelDir);
		File nestedDir = insideNestedDir.getParentFile();

		// create a file in the first-level and two files in the nested dir
		TestFileUtils.createTempFileInDirectory(nestedDir.getAbsolutePath(), "paella");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "kalamari");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "fideua");

		this.format.setFilePath(new Path(nestedDir.toURI().toString()));
		this.config.setBoolean("recursive.file.enumeration", false);
		format.configure(this.config);

		FileInputSplit[] splits = format.createInputSplits(1);
		Assert.assertEquals(1, splits.length);
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
Example #13
Source File: DelimitedInputFormatSamplingTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testDifferentDelimiter() {
	try {
		final String DELIMITER = "12345678-";
		String testData = TEST_DATA1.replace("\n", DELIMITER);
		
		final String tempFile = TestFileUtils.createTempFile(testData);
		final Configuration conf = new Configuration();
		
		final TestDelimitedInputFormat format = new TestDelimitedInputFormat(CONFIG);
		format.setFilePath(tempFile);
		format.setDelimiter(DELIMITER);
		format.configure(conf);
		
		BaseStatistics stats = format.getStatistics(null);
		final int numLines = TEST_DATA_1_LINES;
		final float avgWidth = ((float) testData.length()) / TEST_DATA_1_LINES;
		
		Assert.assertTrue("Wrong record count.", stats.getNumberOfRecords() < numLines + 1 & stats.getNumberOfRecords() > numLines - 1);
		Assert.assertTrue("Wrong avg record size.", stats.getAverageRecordWidth() < avgWidth + 1 & stats.getAverageRecordWidth() > avgWidth - 1);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
Example #14
Source File: EnumerateNestedFilesTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testGetStatisticsOneFileInNestedDir() {
	try {
		final long SIZE = 1024 * 500;
		String firstLevelDir = TestFileUtils.randomFileName();
		String secondLevelDir = TestFileUtils.randomFileName();

		File insideNestedDir = tempFolder.newFolder(firstLevelDir, secondLevelDir);
		File nestedDir = insideNestedDir.getParentFile();

		// create a file in the nested dir
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), SIZE);

		this.format.setFilePath(new Path(nestedDir.toURI().toString()));
		this.config.setBoolean("recursive.file.enumeration", true);
		format.configure(this.config);

		BaseStatistics stats = format.getStatistics(null);
		Assert.assertEquals("The file size from the statistics is wrong.", SIZE, stats.getTotalInputSize());
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
Example #15
Source File: DelimitedInputFormatSamplingTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testSamplingOneFile() {
	try {
		final String tempFile = TestFileUtils.createTempFile(TEST_DATA1);
		final Configuration conf = new Configuration();
		
		final TestDelimitedInputFormat format = new TestDelimitedInputFormat(CONFIG);
		format.setFilePath(tempFile);
		format.configure(conf);
		BaseStatistics stats = format.getStatistics(null);
		
		final int numLines = TEST_DATA_1_LINES;
		final float avgWidth = ((float) TEST_DATA1.length()) / TEST_DATA_1_LINES;
		Assert.assertTrue("Wrong record count.", stats.getNumberOfRecords() < numLines + 1 & stats.getNumberOfRecords() > numLines - 1);
		Assert.assertTrue("Wrong avg record size.", stats.getAverageRecordWidth() < avgWidth + 1 & stats.getAverageRecordWidth() > avgWidth - 1);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
Example #16
Source File: EnumerateNestedFilesTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testGetStatisticsOneFileInNestedDir() {
	try {
		final long SIZE = 1024 * 500;
		String firstLevelDir = TestFileUtils.randomFileName();
		String secondLevelDir = TestFileUtils.randomFileName();

		File insideNestedDir = tempFolder.newFolder(firstLevelDir, secondLevelDir);
		File nestedDir = insideNestedDir.getParentFile();

		// create a file in the nested dir
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), SIZE);

		this.format.setFilePath(new Path(nestedDir.toURI().toString()));
		this.config.setBoolean("recursive.file.enumeration", true);
		format.configure(this.config);

		BaseStatistics stats = format.getStatistics(null);
		Assert.assertEquals("The file size from the statistics is wrong.", SIZE, stats.getTotalInputSize());
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
Example #17
Source File: FileInputFormatTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testGetStatisticsMultipleFilesNoCachedVersion() {
	try {
		final long SIZE1 = 2077;
		final long SIZE2 = 31909;
		final long SIZE3 = 10;
		final long TOTAL = SIZE1 + SIZE2 + SIZE3;
		
		String tempDir = TestFileUtils.createTempFileDir(temporaryFolder.newFolder(), SIZE1, SIZE2, SIZE3);
		
		final DummyFileInputFormat format = new DummyFileInputFormat();
		format.setFilePath(tempDir);
		format.configure(new Configuration());
		
		BaseStatistics stats = format.getStatistics(null);
		Assert.assertEquals("The file size from the statistics is wrong.", TOTAL, stats.getTotalInputSize());
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
Example #18
Source File: FileInputFormatTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testGetStatisticsMultipleFilesMultiplePathsNoCachedVersion() throws IOException {
	final long size1 = 2077;
	final long size2 = 31909;
	final long size3 = 10;
	final long totalSize123 = size1 + size2 + size3;
	
	String tempDir = TestFileUtils.createTempFileDir(temporaryFolder.newFolder(), size1, size2, size3);
	
	final long size4 = 2051;
	final long size5 = 31902;
	final long size6 = 15;
	final long totalSize456 = size4 + size5 + size6;
	String tempDir2 = TestFileUtils.createTempFileDir(temporaryFolder.newFolder(), size4, size5, size6);

	final MultiDummyFileInputFormat format = new MultiDummyFileInputFormat();
	format.setFilePaths(tempDir, tempDir2);
	format.configure(new Configuration());
	
	BaseStatistics stats = format.getStatistics(null);
	Assert.assertEquals("The file size from the statistics is wrong.", totalSize123 + totalSize456, stats.getTotalInputSize());
}
Example #19
Source File: FileInputFormatTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testGetStatisticsMultipleOneFileNoCachedVersion() throws IOException {
	final long size1 = 1024 * 500;
	String tempFile = TestFileUtils.createTempFile(size1);

	final long size2 = 1024 * 505;
	String tempFile2 = TestFileUtils.createTempFile(size2);

	final long totalSize = size1 + size2;
	
	final MultiDummyFileInputFormat format = new MultiDummyFileInputFormat();
	format.setFilePaths(tempFile, tempFile2);
	format.configure(new Configuration());
	
	BaseStatistics stats = format.getStatistics(null);
	Assert.assertEquals("The file size from the statistics is wrong.", totalSize, stats.getTotalInputSize());
}
Example #20
Source File: FileInputFormatTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testGetStatisticsMultipleOneFileNoCachedVersion() throws IOException {
	final long size1 = 1024 * 500;
	String tempFile = TestFileUtils.createTempFile(size1);

	final long size2 = 1024 * 505;
	String tempFile2 = TestFileUtils.createTempFile(size2);

	final long totalSize = size1 + size2;
	
	final MultiDummyFileInputFormat format = new MultiDummyFileInputFormat();
	format.setFilePaths(tempFile, tempFile2);
	format.configure(new Configuration());
	
	BaseStatistics stats = format.getStatistics(null);
	Assert.assertEquals("The file size from the statistics is wrong.", totalSize, stats.getTotalInputSize());
}
Example #21
Source File: FileInputFormatTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testGetStatisticsMultipleFilesMultiplePathsNoCachedVersion() throws IOException {
	final long size1 = 2077;
	final long size2 = 31909;
	final long size3 = 10;
	final long totalSize123 = size1 + size2 + size3;
	
	String tempDir = TestFileUtils.createTempFileDir(temporaryFolder.newFolder(), size1, size2, size3);
	
	final long size4 = 2051;
	final long size5 = 31902;
	final long size6 = 15;
	final long totalSize456 = size4 + size5 + size6;
	String tempDir2 = TestFileUtils.createTempFileDir(temporaryFolder.newFolder(), size4, size5, size6);

	final MultiDummyFileInputFormat format = new MultiDummyFileInputFormat();
	format.setFilePaths(tempDir, tempDir2);
	format.configure(new Configuration());
	
	BaseStatistics stats = format.getStatistics(null);
	Assert.assertEquals("The file size from the statistics is wrong.", totalSize123 + totalSize456, stats.getTotalInputSize());
}
Example #22
Source File: FileInputFormatTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testGetStatisticsMultipleFilesNoCachedVersion() {
	try {
		final long SIZE1 = 2077;
		final long SIZE2 = 31909;
		final long SIZE3 = 10;
		final long TOTAL = SIZE1 + SIZE2 + SIZE3;
		
		String tempDir = TestFileUtils.createTempFileDir(temporaryFolder.newFolder(), SIZE1, SIZE2, SIZE3);
		
		final DummyFileInputFormat format = new DummyFileInputFormat();
		format.setFilePath(tempDir);
		format.configure(new Configuration());
		
		BaseStatistics stats = format.getStatistics(null);
		Assert.assertEquals("The file size from the statistics is wrong.", TOTAL, stats.getTotalInputSize());
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
Example #23
Source File: FileInputFormatTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testGetStatisticsOneFileNoCachedVersion() {
	try {
		final long SIZE = 1024 * 500;
		String tempFile = TestFileUtils.createTempFile(SIZE);
		
		final DummyFileInputFormat format = new DummyFileInputFormat();
		format.setFilePath(tempFile);
		format.configure(new Configuration());
		
		BaseStatistics stats = format.getStatistics(null);
		Assert.assertEquals("The file size from the statistics is wrong.", SIZE, stats.getTotalInputSize());
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
Example #24
Source File: DelimitedInputFormatSamplingTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testSamplingOneFile() {
	try {
		final String tempFile = TestFileUtils.createTempFile(TEST_DATA1);
		final Configuration conf = new Configuration();
		
		final TestDelimitedInputFormat format = new TestDelimitedInputFormat(CONFIG);
		format.setFilePath(tempFile);
		format.configure(conf);
		BaseStatistics stats = format.getStatistics(null);
		
		final int numLines = TEST_DATA_1_LINES;
		final float avgWidth = ((float) TEST_DATA1.length()) / TEST_DATA_1_LINES;
		Assert.assertTrue("Wrong record count.", stats.getNumberOfRecords() < numLines + 1 & stats.getNumberOfRecords() > numLines - 1);
		Assert.assertTrue("Wrong avg record size.", stats.getAverageRecordWidth() < avgWidth + 1 & stats.getAverageRecordWidth() > avgWidth - 1);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
Example #25
Source File: DelimitedInputFormatSamplingTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testDifferentDelimiter() {
	try {
		final String DELIMITER = "12345678-";
		String testData = TEST_DATA1.replace("\n", DELIMITER);
		
		final String tempFile = TestFileUtils.createTempFile(testData);
		final Configuration conf = new Configuration();
		
		final TestDelimitedInputFormat format = new TestDelimitedInputFormat(CONFIG);
		format.setFilePath(tempFile);
		format.setDelimiter(DELIMITER);
		format.configure(conf);
		
		BaseStatistics stats = format.getStatistics(null);
		final int numLines = TEST_DATA_1_LINES;
		final float avgWidth = ((float) testData.length()) / TEST_DATA_1_LINES;
		
		Assert.assertTrue("Wrong record count.", stats.getNumberOfRecords() < numLines + 1 & stats.getNumberOfRecords() > numLines - 1);
		Assert.assertTrue("Wrong avg record size.", stats.getAverageRecordWidth() < avgWidth + 1 & stats.getAverageRecordWidth() > avgWidth - 1);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
Example #26
Source File: DelimitedInputFormatSamplingTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testSamplingOverlyLongRecord() {
	try {
		final String tempFile = TestFileUtils.createTempFile(2 * OptimizerOptions.DELIMITED_FORMAT_MAX_SAMPLE_LEN.defaultValue());
		final Configuration conf = new Configuration();
		
		final TestDelimitedInputFormat format = new TestDelimitedInputFormat(CONFIG);
		format.setFilePath(tempFile);
		format.configure(conf);
		
		Assert.assertNull("Expected exception due to overly long record.", format.getStatistics(null));
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
Example #27
Source File: EnumerateNestedFilesTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testGetStatisticsOneFileInNestedDir() {
	try {
		final long SIZE = 1024 * 500;
		String firstLevelDir = TestFileUtils.randomFileName();
		String secondLevelDir = TestFileUtils.randomFileName();

		File insideNestedDir = tempFolder.newFolder(firstLevelDir, secondLevelDir);
		File nestedDir = insideNestedDir.getParentFile();

		// create a file in the nested dir
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), SIZE);

		this.format.setFilePath(new Path(nestedDir.toURI().toString()));
		this.config.setBoolean("recursive.file.enumeration", true);
		format.configure(this.config);

		BaseStatistics stats = format.getStatistics(null);
		Assert.assertEquals("The file size from the statistics is wrong.", SIZE, stats.getTotalInputSize());
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
Example #28
Source File: EnumerateNestedFilesTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Test without nested directory and recursive.file.enumeration = true
 */
@Test
public void testNoNestedDirectoryTrue() {
	try {
		String filePath = TestFileUtils.createTempFile("foo");

		this.format.setFilePath(new Path(filePath));
		this.config.setBoolean("recursive.file.enumeration", true);
		format.configure(this.config);

		FileInputSplit[] splits = format.createInputSplits(1);
		Assert.assertEquals(1, splits.length);
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
Example #29
Source File: EnumerateNestedFilesTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Test with one nested directory and recursive.file.enumeration = true
 */
@Test
public void testOneNestedDirectoryTrue() {
	try {
		String firstLevelDir = TestFileUtils.randomFileName();
		String secondLevelDir = TestFileUtils.randomFileName();

		File insideNestedDir = tempFolder.newFolder(firstLevelDir, secondLevelDir);
		File nestedDir = insideNestedDir.getParentFile();

		// create a file in the first-level and two files in the nested dir
		TestFileUtils.createTempFileInDirectory(nestedDir.getAbsolutePath(), "paella");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "kalamari");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "fideua");

		this.format.setFilePath(new Path(nestedDir.toURI().toString()));
		this.config.setBoolean("recursive.file.enumeration", true);
		format.configure(this.config);

		FileInputSplit[] splits = format.createInputSplits(1);
		Assert.assertEquals(3, splits.length);
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}
Example #30
Source File: EnumerateNestedFilesTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Test with one nested directory and recursive.file.enumeration = false
 */
@Test
public void testOneNestedDirectoryFalse() {
	try {
		String firstLevelDir = TestFileUtils.randomFileName();
		String secondLevelDir = TestFileUtils.randomFileName();

		File insideNestedDir = tempFolder.newFolder(firstLevelDir, secondLevelDir);
		File nestedDir = insideNestedDir.getParentFile();

		// create a file in the first-level and two files in the nested dir
		TestFileUtils.createTempFileInDirectory(nestedDir.getAbsolutePath(), "paella");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "kalamari");
		TestFileUtils.createTempFileInDirectory(insideNestedDir.getAbsolutePath(), "fideua");

		this.format.setFilePath(new Path(nestedDir.toURI().toString()));
		this.config.setBoolean("recursive.file.enumeration", false);
		format.configure(this.config);

		FileInputSplit[] splits = format.createInputSplits(1);
		Assert.assertEquals(1, splits.length);
	} catch (Exception ex) {
		ex.printStackTrace();
		Assert.fail(ex.getMessage());
	}
}