Java Code Examples for org.apache.flink.core.fs.FSDataInputStream#seek()

The following examples show how to use org.apache.flink.core.fs.FSDataInputStream#seek() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: FileReadFunction.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
public void flatMap(Tuple3<String, Long, Long> value, Collector<String> out) throws Exception {
	FSDataInputStream stream = FileSystem.get(new URI(value.f0)).open(new Path(value.f0));
	stream.seek(value.f1);

	BufferedReader reader = new BufferedReader(new InputStreamReader(stream));
	String line;

	try {
		while ((line = reader.readLine()) != null && (value.f2 == -1L || stream.getPos() <= value.f2)) {
			out.collect(line);
		}
	} finally {
		reader.close();
	}
}
Example 2
Source File: FileReadFunction.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
public void flatMap(Tuple3<String, Long, Long> value, Collector<String> out) throws Exception {
	FSDataInputStream stream = FileSystem.get(new URI(value.f0)).open(new Path(value.f0));
	stream.seek(value.f1);

	BufferedReader reader = new BufferedReader(new InputStreamReader(stream));
	String line;

	try {
		while ((line = reader.readLine()) != null && (value.f2 == -1L || stream.getPos() <= value.f2)) {
			out.collect(line);
		}
	} finally {
		reader.close();
	}
}
Example 3
Source File: ByteStreamStateHandleTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testBulkRead() throws IOException {
	final byte[] data = {34, 25, 22, 66};
	final ByteStreamStateHandle handle = new ByteStreamStateHandle("name", data);
	final int targetLen = 8;

	for (int start = 0; start < data.length; start++) {
		for (int num = 0; num < targetLen; num++) {
			FSDataInputStream in = handle.openInputStream();
			in.seek(start);

			final byte[] target = new byte[targetLen];
			final int read = in.read(target, targetLen - num, num);

			assertEquals(Math.min(num, data.length - start), read);
			for (int i = 0; i < read; i++) {
				assertEquals(data[start + i], target[targetLen - num + i]);
			}

			int newPos = start + read;
			assertEquals(newPos, (int) in.getPos());
			assertEquals(newPos < data.length ? data[newPos] : -1, in.read());
		}
	}
}
Example 4
Source File: OperatorStateRestoreOperation.java    From flink with Apache License 2.0 6 votes vote down vote up 
private <K, V> void deserializeBroadcastStateValues(
	final BackendWritableBroadcastState<K, V> broadcastStateForName,
	final FSDataInputStream in,
	final OperatorStateHandle.StateMetaInfo metaInfo) throws Exception {

	if (metaInfo != null) {
		long[] offsets = metaInfo.getOffsets();
		if (offsets != null) {

			TypeSerializer<K> keySerializer = broadcastStateForName.getStateMetaInfo().getKeySerializer();
			TypeSerializer<V> valueSerializer = broadcastStateForName.getStateMetaInfo().getValueSerializer();

			in.seek(offsets[0]);

			DataInputView div = new DataInputViewStreamWrapper(in);
			int size = div.readInt();
			for (int i = 0; i < size; i++) {
				broadcastStateForName.put(keySerializer.deserialize(div), valueSerializer.deserialize(div));
			}
		}
	}
}
Example 5
Source File: OperatorStateRestoreOperation.java    From flink with Apache License 2.0 6 votes vote down vote up 
private <S> void deserializeOperatorStateValues(
	PartitionableListState<S> stateListForName,
	FSDataInputStream in,
	OperatorStateHandle.StateMetaInfo metaInfo) throws IOException {

	if (null != metaInfo) {
		long[] offsets = metaInfo.getOffsets();
		if (null != offsets) {
			DataInputView div = new DataInputViewStreamWrapper(in);
			TypeSerializer<S> serializer = stateListForName.getStateMetaInfo().getPartitionStateSerializer();
			for (long offset : offsets) {
				in.seek(offset);
				stateListForName.add(serializer.deserialize(div));
			}
		}
	}
}
Example 6
Source File: LineReader.java    From incubator-retired-mrql with Apache License 2.0 6 votes vote down vote up 
public LineReader(final FSDataInputStream strm, final long start, final long length, final int buffersize)
		throws IOException {
	this.stream = strm;
	this.readBuffer = new byte[buffersize];
	this.wrapBuffer = new byte[256];

	this.lengthLeft = length;
	this.readPos = 0;
	this.overLimit = false;

	if (start != 0) {
		strm.seek(start);
		readLine();
	} else {
		fillBuffer();
	}
}
Example 7
Source File: OperatorStateRestoreOperation.java    From flink with Apache License 2.0 6 votes vote down vote up 
private <K, V> void deserializeBroadcastStateValues(
	final BackendWritableBroadcastState<K, V> broadcastStateForName,
	final FSDataInputStream in,
	final OperatorStateHandle.StateMetaInfo metaInfo) throws Exception {

	if (metaInfo != null) {
		long[] offsets = metaInfo.getOffsets();
		if (offsets != null) {

			TypeSerializer<K> keySerializer = broadcastStateForName.getStateMetaInfo().getKeySerializer();
			TypeSerializer<V> valueSerializer = broadcastStateForName.getStateMetaInfo().getValueSerializer();

			in.seek(offsets[0]);

			DataInputView div = new DataInputViewStreamWrapper(in);
			int size = div.readInt();
			for (int i = 0; i < size; i++) {
				broadcastStateForName.put(keySerializer.deserialize(div), valueSerializer.deserialize(div));
			}
		}
	}
}
Example 8
Source File: OperatorStateRestoreOperation.java    From flink with Apache License 2.0 6 votes vote down vote up 
private <S> void deserializeOperatorStateValues(
	PartitionableListState<S> stateListForName,
	FSDataInputStream in,
	OperatorStateHandle.StateMetaInfo metaInfo) throws IOException {

	if (null != metaInfo) {
		long[] offsets = metaInfo.getOffsets();
		if (null != offsets) {
			DataInputView div = new DataInputViewStreamWrapper(in);
			TypeSerializer<S> serializer = stateListForName.getStateMetaInfo().getPartitionStateSerializer();
			for (long offset : offsets) {
				in.seek(offset);
				stateListForName.add(serializer.deserialize(div));
			}
		}
	}
}
Example 9
Source File: FileReadFunction.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Override
public void flatMap(Tuple3<String, Long, Long> value, Collector<String> out) throws Exception {
	FSDataInputStream stream = FileSystem.get(new URI(value.f0)).open(new Path(value.f0));
	stream.seek(value.f1);

	BufferedReader reader = new BufferedReader(new InputStreamReader(stream));
	String line;

	try {
		while ((line = reader.readLine()) != null && (value.f2 == -1L || stream.getPos() <= value.f2)) {
			out.collect(line);
		}
	} finally {
		reader.close();
	}
}
Example 10
Source File: ByteStreamStateHandleTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testBulkRead() throws IOException {
	final byte[] data = {34, 25, 22, 66};
	final ByteStreamStateHandle handle = new ByteStreamStateHandle("name", data);
	final int targetLen = 8;

	for (int start = 0; start < data.length; start++) {
		for (int num = 0; num < targetLen; num++) {
			FSDataInputStream in = handle.openInputStream();
			in.seek(start);

			final byte[] target = new byte[targetLen];
			final int read = in.read(target, targetLen - num, num);

			assertEquals(Math.min(num, data.length - start), read);
			for (int i = 0; i < read; i++) {
				assertEquals(data[start + i], target[targetLen - num + i]);
			}

			int newPos = start + read;
			assertEquals(newPos, (int) in.getPos());
			assertEquals(newPos < data.length ? data[newPos] : -1, in.read());
		}
	}
}
Example 11
Source File: OperatorStateRestoreOperation.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
private <K, V> void deserializeBroadcastStateValues(
	final BackendWritableBroadcastState<K, V> broadcastStateForName,
	final FSDataInputStream in,
	final OperatorStateHandle.StateMetaInfo metaInfo) throws Exception {

	if (metaInfo != null) {
		long[] offsets = metaInfo.getOffsets();
		if (offsets != null) {

			TypeSerializer<K> keySerializer = broadcastStateForName.getStateMetaInfo().getKeySerializer();
			TypeSerializer<V> valueSerializer = broadcastStateForName.getStateMetaInfo().getValueSerializer();

			in.seek(offsets[0]);

			DataInputView div = new DataInputViewStreamWrapper(in);
			int size = div.readInt();
			for (int i = 0; i < size; i++) {
				broadcastStateForName.put(keySerializer.deserialize(div), valueSerializer.deserialize(div));
			}
		}
	}
}
Example 12
Source File: OperatorStateRestoreOperation.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
private <S> void deserializeOperatorStateValues(
	PartitionableListState<S> stateListForName,
	FSDataInputStream in,
	OperatorStateHandle.StateMetaInfo metaInfo) throws IOException {

	if (null != metaInfo) {
		long[] offsets = metaInfo.getOffsets();
		if (null != offsets) {
			DataInputView div = new DataInputViewStreamWrapper(in);
			TypeSerializer<S> serializer = stateListForName.getStateMetaInfo().getPartitionStateSerializer();
			for (long offset : offsets) {
				in.seek(offset);
				stateListForName.add(serializer.deserialize(div));
			}
		}
	}
}
Example 13
Source File: HeapRestoreOperation.java    From flink with Apache License 2.0 5 votes vote down vote up 
private void readStateHandleStateData(
	FSDataInputStream fsDataInputStream,
	DataInputViewStreamWrapper inView,
	KeyGroupRangeOffsets keyGroupOffsets,
	Map<Integer, StateMetaInfoSnapshot> kvStatesById,
	int numStates,
	int readVersion,
	boolean isCompressed) throws IOException {

	final StreamCompressionDecorator streamCompressionDecorator = isCompressed ?
		SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE;

	for (Tuple2<Integer, Long> groupOffset : keyGroupOffsets) {
		int keyGroupIndex = groupOffset.f0;
		long offset = groupOffset.f1;

		// Check that restored key groups all belong to the backend.
		Preconditions.checkState(keyGroupRange.contains(keyGroupIndex), "The key group must belong to the backend.");

		fsDataInputStream.seek(offset);

		int writtenKeyGroupIndex = inView.readInt();
		Preconditions.checkState(writtenKeyGroupIndex == keyGroupIndex,
			"Unexpected key-group in restore.");

		try (InputStream kgCompressionInStream =
				 streamCompressionDecorator.decorateWithCompression(fsDataInputStream)) {

			readKeyGroupStateData(
				kgCompressionInStream,
				kvStatesById,
				keyGroupIndex,
				numStates,
				readVersion);
		}
	}
}
Example 14
Source File: StateInitializationContextImplTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public FSDataInputStream openInputStream() throws IOException {
	final FSDataInputStream original = super.openInputStream();

	return new FSDataInputStream() {

		private boolean closed = false;

		@Override
		public void seek(long desired) throws IOException {
			original.seek(desired);
		}

		@Override
		public long getPos() throws IOException {
			return original.getPos();
		}

		@Override
		public int read() throws IOException {
			if (closed) {
				throw new IOException("Stream closed");
			}
			return original.read();
		}

		@Override
		public void close() throws IOException {
			original.close();
			this.closed = true;
		}
	};
}
Example 15
Source File: HeapRestoreOperation.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
private void readStateHandleStateData(
	FSDataInputStream fsDataInputStream,
	DataInputViewStreamWrapper inView,
	KeyGroupRangeOffsets keyGroupOffsets,
	Map<Integer, StateMetaInfoSnapshot> kvStatesById,
	int numStates,
	int readVersion,
	boolean isCompressed) throws IOException {

	final StreamCompressionDecorator streamCompressionDecorator = isCompressed ?
		SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE;

	for (Tuple2<Integer, Long> groupOffset : keyGroupOffsets) {
		int keyGroupIndex = groupOffset.f0;
		long offset = groupOffset.f1;

		// Check that restored key groups all belong to the backend.
		Preconditions.checkState(keyGroupRange.contains(keyGroupIndex), "The key group must belong to the backend.");

		fsDataInputStream.seek(offset);

		int writtenKeyGroupIndex = inView.readInt();
		Preconditions.checkState(writtenKeyGroupIndex == keyGroupIndex,
			"Unexpected key-group in restore.");

		try (InputStream kgCompressionInStream =
				 streamCompressionDecorator.decorateWithCompression(fsDataInputStream)) {

			readKeyGroupStateData(
				kgCompressionInStream,
				kvStatesById,
				keyGroupIndex,
				numStates,
				readVersion);
		}
	}
}
Example 16
Source File: StateInitializationContextImplTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public FSDataInputStream openInputStream() throws IOException {
	final FSDataInputStream original = super.openInputStream();

	return new FSDataInputStream() {

		private boolean closed = false;

		@Override
		public void seek(long desired) throws IOException {
			original.seek(desired);
		}

		@Override
		public long getPos() throws IOException {
			return original.getPos();
		}

		@Override
		public int read() throws IOException {
			if (closed) {
				throw new IOException("Stream closed");
			}
			return original.read();
		}

		@Override
		public void close() throws IOException {
			original.close();
			this.closed = true;
		}
	};
}
Example 17
Source File: FlinkXMLSplitter.java    From incubator-retired-mrql with Apache License 2.0 5 votes vote down vote up 
FlinkXMLSplitter ( String[] tags, FSDataInputStream fsin, long start, long end ) {
    in_memory = false;
    this.tags = tags;
    this.fsin = fsin;
    this.start = start;
    this.end = end;
    pos = start;
    this.buffer = new DataOutputBuffer();
    try {
        fsin.seek(start);
    } catch ( IOException e ) {
        System.err.println("*** Cannot parse the data split: "+fsin);
    }
}
Example 18
Source File: FlinkJsonSplitter.java    From incubator-retired-mrql with Apache License 2.0 5 votes vote down vote up 
FlinkJsonSplitter ( String[] tags, FSDataInputStream fsin, long start, long end ) {
    in_memory = false;
    this.tags = tags;
    this.fsin = fsin;
    this.end = end;
    this.buffer = new DataOutputBuffer();
    try {
        fsin.seek(start);
        this.start = (start == 0) ? start : sync(start);
        fsin.seek(this.start);
        scanner = new JSONLex(fsin);
    } catch ( IOException e ) {
        System.err.println("*** Cannot parse the data split: "+fsin);
    }
}
Example 19
Source File: StateInitializationContextImplTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
public FSDataInputStream openInputStream() throws IOException {
	final FSDataInputStream original = super.openInputStream();

	return new FSDataInputStream() {

		private boolean closed = false;

		@Override
		public void seek(long desired) throws IOException {
			original.seek(desired);
		}

		@Override
		public long getPos() throws IOException {
			return original.getPos();
		}

		@Override
		public int read() throws IOException {
			if (closed) {
				throw new IOException("Stream closed");
			}
			return original.read();
		}

		@Override
		public void close() throws IOException {
			original.close();
			this.closed = true;
		}
	};
}
Example 20
Source File: HeapRestoreOperation.java    From flink with Apache License 2.0 5 votes vote down vote up 
private void readStateHandleStateData(
	FSDataInputStream fsDataInputStream,
	DataInputViewStreamWrapper inView,
	KeyGroupRangeOffsets keyGroupOffsets,
	Map<Integer, StateMetaInfoSnapshot> kvStatesById,
	int numStates,
	int readVersion,
	boolean isCompressed) throws IOException {

	final StreamCompressionDecorator streamCompressionDecorator = isCompressed ?
		SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE;

	for (Tuple2<Integer, Long> groupOffset : keyGroupOffsets) {
		int keyGroupIndex = groupOffset.f0;
		long offset = groupOffset.f1;

		// Check that restored key groups all belong to the backend.
		Preconditions.checkState(keyGroupRange.contains(keyGroupIndex), "The key group must belong to the backend.");

		fsDataInputStream.seek(offset);

		int writtenKeyGroupIndex = inView.readInt();
		Preconditions.checkState(writtenKeyGroupIndex == keyGroupIndex,
			"Unexpected key-group in restore.");

		try (InputStream kgCompressionInStream =
				 streamCompressionDecorator.decorateWithCompression(fsDataInputStream)) {

			readKeyGroupStateData(
				kgCompressionInStream,
				kvStatesById,
				keyGroupIndex,
				numStates,
				readVersion);
		}
	}
}