Java Code Examples for org.apache.flink.core.fs.FileInputSplit#getHostnames()

The following examples show how to use org.apache.flink.core.fs.FileInputSplit#getHostnames() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: DelimitedInputFormatTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
/**
 * Tests that the records are read correctly when the split boundary is in the middle of a record.
 */
@Test
public void testReadOverSplitBoundariesUnaligned() throws IOException {
	final String myString = "value1\nvalue2\nvalue3";
	final FileInputSplit split = createTempFile(myString);

	FileInputSplit split1 = new FileInputSplit(0, split.getPath(), 0, split.getLength() / 2, split.getHostnames());
	FileInputSplit split2 = new FileInputSplit(1, split.getPath(), split1.getLength(), split.getLength(), split.getHostnames());

	final Configuration parameters = new Configuration();

	format.configure(parameters);
	format.open(split1);

	assertEquals("value1", format.nextRecord(null));
	assertEquals("value2", format.nextRecord(null));
	assertNull(format.nextRecord(null));
	assertTrue(format.reachedEnd());

	format.close();
	format.open(split2);

	assertEquals("value3", format.nextRecord(null));
	assertNull(format.nextRecord(null));
	assertTrue(format.reachedEnd());

	format.close();
}
Example 2
Source File: DelimitedInputFormatTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Test
public void testReadRecordsLargerThanBuffer() throws IOException {
	final String myString = "aaaaaaaaaaaaaaaaaaaaa\n" +
							"bbbbbbbbbbbbbbbbbbbbbbbbb\n" +
							"ccccccccccccccccccc\n" +
							"ddddddddddddddddddddddddddddddddddd\n";

	final FileInputSplit split = createTempFile(myString);
	FileInputSplit split1 = new FileInputSplit(0, split.getPath(), 0, split.getLength() / 2, split.getHostnames());
	FileInputSplit split2 = new FileInputSplit(1, split.getPath(), split1.getLength(), split.getLength(), split.getHostnames());

	final Configuration parameters = new Configuration();

	format.setBufferSize(8);
	format.configure(parameters);

	String next;
	List<String> result = new ArrayList<String>();


	format.open(split1);
	while ((next = format.nextRecord(null)) != null) {
		result.add(next);
	}
	assertNull(format.nextRecord(null));
	assertTrue(format.reachedEnd());
	format.close();

	format.open(split2);
	while ((next = format.nextRecord(null)) != null) {
		result.add(next);
	}
	assertNull(format.nextRecord(null));
	assertTrue(format.reachedEnd());
	format.close();

	assertEquals(4, result.size());
	assertEquals(Arrays.asList(myString.split("\n")), result);
}
Example 3
Source File: ContinuousFileProcessingRescalingTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
private TimestampedFileInputSplit getTimestampedSplit(long modTime, FileInputSplit split) {
	Preconditions.checkNotNull(split);
	return new TimestampedFileInputSplit(
		modTime,
		split.getSplitNumber(),
		split.getPath(),
		split.getStart(),
		split.getLength(),
		split.getHostnames());
}
Example 4
Source File: DelimitedInputFormatTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Tests that the records are read correctly when the split boundary is in the middle of a record.
 */
@Test
public void testReadOverSplitBoundariesUnaligned() throws IOException {
	final String myString = "value1\nvalue2\nvalue3";
	final FileInputSplit split = createTempFile(myString);

	FileInputSplit split1 = new FileInputSplit(0, split.getPath(), 0, split.getLength() / 2, split.getHostnames());
	FileInputSplit split2 = new FileInputSplit(1, split.getPath(), split1.getLength(), split.getLength(), split.getHostnames());

	final Configuration parameters = new Configuration();

	format.configure(parameters);
	format.open(split1);

	assertEquals("value1", format.nextRecord(null));
	assertEquals("value2", format.nextRecord(null));
	assertNull(format.nextRecord(null));
	assertTrue(format.reachedEnd());

	format.close();
	format.open(split2);

	assertEquals("value3", format.nextRecord(null));
	assertNull(format.nextRecord(null));
	assertTrue(format.reachedEnd());

	format.close();
}
Example 5
Source File: DelimitedInputFormatTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testReadRecordsLargerThanBuffer() throws IOException {
	final String myString = "aaaaaaaaaaaaaaaaaaaaa\n" +
							"bbbbbbbbbbbbbbbbbbbbbbbbb\n" +
							"ccccccccccccccccccc\n" +
							"ddddddddddddddddddddddddddddddddddd\n";

	final FileInputSplit split = createTempFile(myString);
	FileInputSplit split1 = new FileInputSplit(0, split.getPath(), 0, split.getLength() / 2, split.getHostnames());
	FileInputSplit split2 = new FileInputSplit(1, split.getPath(), split1.getLength(), split.getLength(), split.getHostnames());

	final Configuration parameters = new Configuration();

	format.setBufferSize(8);
	format.configure(parameters);

	String next;
	List<String> result = new ArrayList<String>();


	format.open(split1);
	while ((next = format.nextRecord(null)) != null) {
		result.add(next);
	}
	assertNull(format.nextRecord(null));
	assertTrue(format.reachedEnd());
	format.close();

	format.open(split2);
	while ((next = format.nextRecord(null)) != null) {
		result.add(next);
	}
	assertNull(format.nextRecord(null));
	assertTrue(format.reachedEnd());
	format.close();

	assertEquals(4, result.size());
	assertEquals(Arrays.asList(myString.split("\n")), result);
}
Example 6
Source File: ContinuousFileProcessingRescalingTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
private TimestampedFileInputSplit getTimestampedSplit(long modTime, FileInputSplit split) {
	Preconditions.checkNotNull(split);
	return new TimestampedFileInputSplit(
		modTime,
		split.getSplitNumber(),
		split.getPath(),
		split.getStart(),
		split.getLength(),
		split.getHostnames());
}
Example 7
Source File: DelimitedInputFormatTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Tests that the records are read correctly when the split boundary is in the middle of a record.
 */
@Test
public void testReadOverSplitBoundariesUnaligned() throws IOException {
	final String myString = "value1\nvalue2\nvalue3";
	final FileInputSplit split = createTempFile(myString);

	FileInputSplit split1 = new FileInputSplit(0, split.getPath(), 0, split.getLength() / 2, split.getHostnames());
	FileInputSplit split2 = new FileInputSplit(1, split.getPath(), split1.getLength(), split.getLength(), split.getHostnames());

	final Configuration parameters = new Configuration();

	format.configure(parameters);
	format.open(split1);

	assertEquals("value1", format.nextRecord(null));
	assertEquals("value2", format.nextRecord(null));
	assertNull(format.nextRecord(null));
	assertTrue(format.reachedEnd());

	format.close();
	format.open(split2);

	assertEquals("value3", format.nextRecord(null));
	assertNull(format.nextRecord(null));
	assertTrue(format.reachedEnd());

	format.close();
}
Example 8
Source File: DelimitedInputFormatTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testReadRecordsLargerThanBuffer() throws IOException {
	final String myString = "aaaaaaaaaaaaaaaaaaaaa\n" +
							"bbbbbbbbbbbbbbbbbbbbbbbbb\n" +
							"ccccccccccccccccccc\n" +
							"ddddddddddddddddddddddddddddddddddd\n";

	final FileInputSplit split = createTempFile(myString);
	FileInputSplit split1 = new FileInputSplit(0, split.getPath(), 0, split.getLength() / 2, split.getHostnames());
	FileInputSplit split2 = new FileInputSplit(1, split.getPath(), split1.getLength(), split.getLength(), split.getHostnames());

	final Configuration parameters = new Configuration();

	format.setBufferSize(8);
	format.configure(parameters);

	String next;
	List<String> result = new ArrayList<String>();


	format.open(split1);
	while ((next = format.nextRecord(null)) != null) {
		result.add(next);
	}
	assertNull(format.nextRecord(null));
	assertTrue(format.reachedEnd());
	format.close();

	format.open(split2);
	while ((next = format.nextRecord(null)) != null) {
		result.add(next);
	}
	assertNull(format.nextRecord(null));
	assertTrue(format.reachedEnd());
	format.close();

	assertEquals(4, result.size());
	assertEquals(Arrays.asList(myString.split("\n")), result);
}
Example 9
Source File: ContinuousFileProcessingRescalingTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
private TimestampedFileInputSplit getTimestampedSplit(long modTime, FileInputSplit split) {
	Preconditions.checkNotNull(split);
	return new TimestampedFileInputSplit(
		modTime,
		split.getSplitNumber(),
		split.getPath(),
		split.getStart(),
		split.getLength(),
		split.getHostnames());
}
Example 10
Source File: DelimitedInputFormatTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
/**
 * Tests that the correct number of records is read when the split boundary is exact at the record boundary.
 */
@Test
public void testReadWithBufferSizeIsMultiple() throws IOException {
	final String myString = "aaaaaaa\nbbbbbbb\nccccccc\nddddddd\n";
	final FileInputSplit split = createTempFile(myString);

	FileInputSplit split1 = new FileInputSplit(0, split.getPath(), 0, split.getLength() / 2, split.getHostnames());
	FileInputSplit split2 = new FileInputSplit(1, split.getPath(), split1.getLength(), split.getLength(), split.getHostnames());

	final Configuration parameters = new Configuration();

	format.setBufferSize(2 * ((int) split1.getLength()));
	format.configure(parameters);

	String next;
	int count = 0;

	// read split 1
	format.open(split1);
	while ((next = format.nextRecord(null)) != null) {
		assertEquals(7, next.length());
		count++;
	}
	assertNull(format.nextRecord(null));
	assertTrue(format.reachedEnd());
	format.close();

	// this one must have read one too many, because the next split will skipp the trailing remainder
	// which happens to be one full record
	assertEquals(3, count);

	// read split 2
	format.open(split2);
	while ((next = format.nextRecord(null)) != null) {
		assertEquals(7, next.length());
		count++;
	}
	format.close();

	assertEquals(4, count);
}
Example 11
Source File: DelimitedInputFormatTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * Tests that the correct number of records is read when the split boundary is exact at the record boundary.
 */
@Test
public void testReadWithBufferSizeIsMultiple() throws IOException {
	final String myString = "aaaaaaa\nbbbbbbb\nccccccc\nddddddd\n";
	final FileInputSplit split = createTempFile(myString);

	FileInputSplit split1 = new FileInputSplit(0, split.getPath(), 0, split.getLength() / 2, split.getHostnames());
	FileInputSplit split2 = new FileInputSplit(1, split.getPath(), split1.getLength(), split.getLength(), split.getHostnames());

	final Configuration parameters = new Configuration();

	format.setBufferSize(2 * ((int) split1.getLength()));
	format.configure(parameters);

	String next;
	int count = 0;

	// read split 1
	format.open(split1);
	while ((next = format.nextRecord(null)) != null) {
		assertEquals(7, next.length());
		count++;
	}
	assertNull(format.nextRecord(null));
	assertTrue(format.reachedEnd());
	format.close();

	// this one must have read one too many, because the next split will skipp the trailing remainder
	// which happens to be one full record
	assertEquals(3, count);

	// read split 2
	format.open(split2);
	while ((next = format.nextRecord(null)) != null) {
		assertEquals(7, next.length());
		count++;
	}
	format.close();

	assertEquals(4, count);
}
Example 12
Source File: DelimitedInputFormatTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * Tests that the correct number of records is read when the split boundary is exact at the record boundary.
 */
@Test
public void testReadWithBufferSizeIsMultiple() throws IOException {
	final String myString = "aaaaaaa\nbbbbbbb\nccccccc\nddddddd\n";
	final FileInputSplit split = createTempFile(myString);

	FileInputSplit split1 = new FileInputSplit(0, split.getPath(), 0, split.getLength() / 2, split.getHostnames());
	FileInputSplit split2 = new FileInputSplit(1, split.getPath(), split1.getLength(), split.getLength(), split.getHostnames());

	final Configuration parameters = new Configuration();

	format.setBufferSize(2 * ((int) split1.getLength()));
	format.configure(parameters);

	String next;
	int count = 0;

	// read split 1
	format.open(split1);
	while ((next = format.nextRecord(null)) != null) {
		assertEquals(7, next.length());
		count++;
	}
	assertNull(format.nextRecord(null));
	assertTrue(format.reachedEnd());
	format.close();

	// this one must have read one too many, because the next split will skipp the trailing remainder
	// which happens to be one full record
	assertEquals(3, count);

	// read split 2
	format.open(split2);
	while ((next = format.nextRecord(null)) != null) {
		assertEquals(7, next.length());
		count++;
	}
	format.close();

	assertEquals(4, count);
}