Java Code Examples for org.apache.flink.core.fs.FSDataInputStream#getPos()

The following examples show how to use org.apache.flink.core.fs.FSDataInputStream#getPos() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: BlockingFSDataInputStream.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
public BlockingFSDataInputStream(
	@Nullable FSDataInputStream delegate,
	@Nullable OneShotLatch waitForBlock,
	@Nullable OneShotLatch triggerUnblock,
	long blockAtPosition) {

	this.delegate = delegate;
	this.triggerUnblock = triggerUnblock;
	this.waitUntilStreamBlocked = waitForBlock;
	this.blockAtPosition = blockAtPosition;
	if (delegate != null) {
		try {
			this.position = delegate.getPos();
		} catch (IOException e) {
			throw new RuntimeException(e);
		}
	} else {
		this.position = 0;
	}
	this.closed = new AtomicBoolean(false);
}
Example 2
Source File: FileReadFunction.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Override
public void flatMap(Tuple3<String, Long, Long> value, Collector<String> out) throws Exception {
	FSDataInputStream stream = FileSystem.get(new URI(value.f0)).open(new Path(value.f0));
	stream.seek(value.f1);

	BufferedReader reader = new BufferedReader(new InputStreamReader(stream));
	String line;

	try {
		while ((line = reader.readLine()) != null && (value.f2 == -1L || stream.getPos() <= value.f2)) {
			out.collect(line);
		}
	} finally {
		reader.close();
	}
}
Example 3
Source File: BlockingFSDataInputStream.java    From flink with Apache License 2.0 6 votes vote down vote up 
public BlockingFSDataInputStream(
	@Nullable FSDataInputStream delegate,
	@Nullable OneShotLatch waitForBlock,
	@Nullable OneShotLatch triggerUnblock,
	long blockAtPosition) {

	this.delegate = delegate;
	this.triggerUnblock = triggerUnblock;
	this.waitUntilStreamBlocked = waitForBlock;
	this.blockAtPosition = blockAtPosition;
	if (delegate != null) {
		try {
			this.position = delegate.getPos();
		} catch (IOException e) {
			throw new RuntimeException(e);
		}
	} else {
		this.position = 0;
	}
	this.closed = new AtomicBoolean(false);
}
Example 4
Source File: FileReadFunction.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
public void flatMap(Tuple3<String, Long, Long> value, Collector<String> out) throws Exception {
	FSDataInputStream stream = FileSystem.get(new URI(value.f0)).open(new Path(value.f0));
	stream.seek(value.f1);

	BufferedReader reader = new BufferedReader(new InputStreamReader(stream));
	String line;

	try {
		while ((line = reader.readLine()) != null && (value.f2 == -1L || stream.getPos() <= value.f2)) {
			out.collect(line);
		}
	} finally {
		reader.close();
	}
}
Example 5
Source File: BlockingFSDataInputStream.java    From flink with Apache License 2.0 6 votes vote down vote up 
public BlockingFSDataInputStream(
	@Nullable FSDataInputStream delegate,
	@Nullable OneShotLatch waitForBlock,
	@Nullable OneShotLatch triggerUnblock,
	long blockAtPosition) {

	this.delegate = delegate;
	this.triggerUnblock = triggerUnblock;
	this.waitUntilStreamBlocked = waitForBlock;
	this.blockAtPosition = blockAtPosition;
	if (delegate != null) {
		try {
			this.position = delegate.getPos();
		} catch (IOException e) {
			throw new RuntimeException(e);
		}
	} else {
		this.position = 0;
	}
	this.closed = new AtomicBoolean(false);
}
Example 6
Source File: FileReadFunction.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
public void flatMap(Tuple3<String, Long, Long> value, Collector<String> out) throws Exception {
	FSDataInputStream stream = FileSystem.get(new URI(value.f0)).open(new Path(value.f0));
	stream.seek(value.f1);

	BufferedReader reader = new BufferedReader(new InputStreamReader(stream));
	String line;

	try {
		while ((line = reader.readLine()) != null && (value.f2 == -1L || stream.getPos() <= value.f2)) {
			out.collect(line);
		}
	} finally {
		reader.close();
	}
}
Example 7
Source File: StateInitializationContextImplTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
public FSDataInputStream openInputStream() throws IOException {
	final FSDataInputStream original = super.openInputStream();

	return new FSDataInputStream() {

		private boolean closed = false;

		@Override
		public void seek(long desired) throws IOException {
			original.seek(desired);
		}

		@Override
		public long getPos() throws IOException {
			return original.getPos();
		}

		@Override
		public int read() throws IOException {
			if (closed) {
				throw new IOException("Stream closed");
			}
			return original.read();
		}

		@Override
		public void close() throws IOException {
			original.close();
			this.closed = true;
		}
	};
}
Example 8
Source File: StateInitializationContextImplTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public FSDataInputStream openInputStream() throws IOException {
	final FSDataInputStream original = super.openInputStream();

	return new FSDataInputStream() {

		private boolean closed = false;

		@Override
		public void seek(long desired) throws IOException {
			original.seek(desired);
		}

		@Override
		public long getPos() throws IOException {
			return original.getPos();
		}

		@Override
		public int read() throws IOException {
			if (closed) {
				throw new IOException("Stream closed");
			}
			return original.read();
		}

		@Override
		public void close() throws IOException {
			original.close();
			this.closed = true;
		}
	};
}
Example 9
Source File: StateInitializationContextImplTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public FSDataInputStream openInputStream() throws IOException {
	final FSDataInputStream original = super.openInputStream();

	return new FSDataInputStream() {

		private boolean closed = false;

		@Override
		public void seek(long desired) throws IOException {
			original.seek(desired);
		}

		@Override
		public long getPos() throws IOException {
			return original.getPos();
		}

		@Override
		public int read() throws IOException {
			if (closed) {
				throw new IOException("Stream closed");
			}
			return original.read();
		}

		@Override
		public void close() throws IOException {
			original.close();
			this.closed = true;
		}
	};
}
Example 10
Source File: BoundedInputStream.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
public BoundedInputStream(FSDataInputStream delegate, long endOffsetExclusive) throws IOException {
	this.position = delegate.getPos();
	this.mark = -1L;
	this.endOffsetExclusive = endOffsetExclusive;
	this.delegate = delegate;
}
Example 11
Source File: BoundedInputStream.java    From flink with Apache License 2.0 4 votes vote down vote up 
public BoundedInputStream(FSDataInputStream delegate, long endOffsetExclusive) throws IOException {
	this.position = delegate.getPos();
	this.mark = -1L;
	this.endOffsetExclusive = endOffsetExclusive;
	this.delegate = delegate;
}
Example 12
Source File: BoundedInputStream.java    From flink with Apache License 2.0 4 votes vote down vote up 
public BoundedInputStream(FSDataInputStream delegate, long endOffsetExclusive) throws IOException {
	this.position = delegate.getPos();
	this.mark = -1L;
	this.endOffsetExclusive = endOffsetExclusive;
	this.delegate = delegate;
}