Java Code Examples for org.apache.flink.core.fs.FSDataInputStream#read()

The following examples show how to use org.apache.flink.core.fs.FSDataInputStream#read() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: ByteStreamStateHandleTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testBulkRead() throws IOException {
	final byte[] data = {34, 25, 22, 66};
	final ByteStreamStateHandle handle = new ByteStreamStateHandle("name", data);
	final int targetLen = 8;

	for (int start = 0; start < data.length; start++) {
		for (int num = 0; num < targetLen; num++) {
			FSDataInputStream in = handle.openInputStream();
			in.seek(start);

			final byte[] target = new byte[targetLen];
			final int read = in.read(target, targetLen - num, num);

			assertEquals(Math.min(num, data.length - start), read);
			for (int i = 0; i < read; i++) {
				assertEquals(data[start + i], target[targetLen - num + i]);
			}

			int newPos = start + read;
			assertEquals(newPos, (int) in.getPos());
			assertEquals(newPos < data.length ? data[newPos] : -1, in.read());
		}
	}
}
Example 2
Source File: ByteStreamStateHandleTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testBulkRead() throws IOException {
	final byte[] data = {34, 25, 22, 66};
	final ByteStreamStateHandle handle = new ByteStreamStateHandle("name", data);
	final int targetLen = 8;

	for (int start = 0; start < data.length; start++) {
		for (int num = 0; num < targetLen; num++) {
			FSDataInputStream in = handle.openInputStream();
			in.seek(start);

			final byte[] target = new byte[targetLen];
			final int read = in.read(target, targetLen - num, num);

			assertEquals(Math.min(num, data.length - start), read);
			for (int i = 0; i < read; i++) {
				assertEquals(data[start + i], target[targetLen - num + i]);
			}

			int newPos = start + read;
			assertEquals(newPos, (int) in.getPos());
			assertEquals(newPos < data.length ? data[newPos] : -1, in.read());
		}
	}
}
Example 3
Source File: ByteStreamStateHandleTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testBulkRead() throws IOException {
	final byte[] data = {34, 25, 22, 66};
	final ByteStreamStateHandle handle = new ByteStreamStateHandle("name", data);
	final int targetLen = 8;

	for (int start = 0; start < data.length; start++) {
		for (int num = 0; num < targetLen; num++) {
			FSDataInputStream in = handle.openInputStream();
			in.seek(start);

			final byte[] target = new byte[targetLen];
			final int read = in.read(target, targetLen - num, num);

			assertEquals(Math.min(num, data.length - start), read);
			for (int i = 0; i < read; i++) {
				assertEquals(data[start + i], target[targetLen - num + i]);
			}

			int newPos = start + read;
			assertEquals(newPos, (int) in.getPos());
			assertEquals(newPos < data.length ? data[newPos] : -1, in.read());
		}
	}
}
Example 4
Source File: RocksDBStateDownloader.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
/**
 * Copies the file from a single state handle to the given path.
 */
private void downloadDataForStateHandle(
	Path restoreFilePath,
	StreamStateHandle remoteFileHandle,
	CloseableRegistry closeableRegistry) throws IOException {

	FSDataInputStream inputStream = null;
	FSDataOutputStream outputStream = null;

	try {
		FileSystem restoreFileSystem = restoreFilePath.getFileSystem();
		inputStream = remoteFileHandle.openInputStream();
		closeableRegistry.registerCloseable(inputStream);

		outputStream = restoreFileSystem.create(restoreFilePath, FileSystem.WriteMode.OVERWRITE);
		closeableRegistry.registerCloseable(outputStream);

		byte[] buffer = new byte[8 * 1024];
		while (true) {
			int numBytes = inputStream.read(buffer);
			if (numBytes == -1) {
				break;
			}

			outputStream.write(buffer, 0, numBytes);
		}
	} finally {
		if (closeableRegistry.unregisterCloseable(inputStream)) {
			inputStream.close();
		}

		if (closeableRegistry.unregisterCloseable(outputStream)) {
			outputStream.close();
		}
	}
}
Example 5
Source File: StateInitializationContextImplTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
public FSDataInputStream openInputStream() throws IOException {
	final FSDataInputStream original = super.openInputStream();

	return new FSDataInputStream() {

		private boolean closed = false;

		@Override
		public void seek(long desired) throws IOException {
			original.seek(desired);
		}

		@Override
		public long getPos() throws IOException {
			return original.getPos();
		}

		@Override
		public int read() throws IOException {
			if (closed) {
				throw new IOException("Stream closed");
			}
			return original.read();
		}

		@Override
		public void close() throws IOException {
			original.close();
			this.closed = true;
		}
	};
}
Example 6
Source File: RocksDBStateDownloader.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Copies the file from a single state handle to the given path.
 */
private void downloadDataForStateHandle(
	Path restoreFilePath,
	StreamStateHandle remoteFileHandle,
	CloseableRegistry closeableRegistry) throws IOException {

	FSDataInputStream inputStream = null;
	FSDataOutputStream outputStream = null;

	try {
		FileSystem restoreFileSystem = restoreFilePath.getFileSystem();
		inputStream = remoteFileHandle.openInputStream();
		closeableRegistry.registerCloseable(inputStream);

		outputStream = restoreFileSystem.create(restoreFilePath, FileSystem.WriteMode.OVERWRITE);
		closeableRegistry.registerCloseable(outputStream);

		byte[] buffer = new byte[8 * 1024];
		while (true) {
			int numBytes = inputStream.read(buffer);
			if (numBytes == -1) {
				break;
			}

			outputStream.write(buffer, 0, numBytes);
		}
	} finally {
		if (closeableRegistry.unregisterCloseable(inputStream)) {
			inputStream.close();
		}

		if (closeableRegistry.unregisterCloseable(outputStream)) {
			outputStream.close();
		}
	}
}
Example 7
Source File: StateInitializationContextImplTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public FSDataInputStream openInputStream() throws IOException {
	final FSDataInputStream original = super.openInputStream();

	return new FSDataInputStream() {

		private boolean closed = false;

		@Override
		public void seek(long desired) throws IOException {
			original.seek(desired);
		}

		@Override
		public long getPos() throws IOException {
			return original.getPos();
		}

		@Override
		public int read() throws IOException {
			if (closed) {
				throw new IOException("Stream closed");
			}
			return original.read();
		}

		@Override
		public void close() throws IOException {
			original.close();
			this.closed = true;
		}
	};
}
Example 8
Source File: RocksDBCheckpointIterator.java    From bravo with Apache License 2.0 5 votes vote down vote up 
private void copyStateDataHandleData(
		Path restoreFilePath,
		StreamStateHandle remoteFileHandle) throws IOException {

	FileSystem restoreFileSystem = restoreFilePath.getFileSystem();

	FSDataInputStream inputStream = null;
	FSDataOutputStream outputStream = null;

	try {
		inputStream = remoteFileHandle.openInputStream();
		cancelStreamRegistry.registerCloseable(inputStream);

		outputStream = restoreFileSystem.create(restoreFilePath, FileSystem.WriteMode.OVERWRITE);
		cancelStreamRegistry.registerCloseable(outputStream);

		byte[] buffer = new byte[8 * 1024];
		while (true) {
			int numBytes = inputStream.read(buffer);
			if (numBytes == -1) {
				break;
			}

			outputStream.write(buffer, 0, numBytes);
		}
	} finally {
		if (cancelStreamRegistry.unregisterCloseable(inputStream)) {
			inputStream.close();
		}

		if (cancelStreamRegistry.unregisterCloseable(outputStream)) {
			outputStream.close();
		}
	}
}
Example 9
Source File: RocksDBStateDownloader.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Copies the file from a single state handle to the given path.
 */
private void downloadDataForStateHandle(
	Path restoreFilePath,
	StreamStateHandle remoteFileHandle,
	CloseableRegistry closeableRegistry) throws IOException {

	FSDataInputStream inputStream = null;
	OutputStream outputStream = null;

	try {
		inputStream = remoteFileHandle.openInputStream();
		closeableRegistry.registerCloseable(inputStream);

		Files.createDirectories(restoreFilePath.getParent());
		outputStream = Files.newOutputStream(restoreFilePath);
		closeableRegistry.registerCloseable(outputStream);

		byte[] buffer = new byte[8 * 1024];
		while (true) {
			int numBytes = inputStream.read(buffer);
			if (numBytes == -1) {
				break;
			}

			outputStream.write(buffer, 0, numBytes);
		}
	} finally {
		if (closeableRegistry.unregisterCloseable(inputStream)) {
			inputStream.close();
		}

		if (closeableRegistry.unregisterCloseable(outputStream)) {
			outputStream.close();
		}
	}
}
Example 10
Source File: StateInitializationContextImplTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public FSDataInputStream openInputStream() throws IOException {
	final FSDataInputStream original = super.openInputStream();

	return new FSDataInputStream() {

		private boolean closed = false;

		@Override
		public void seek(long desired) throws IOException {
			original.seek(desired);
		}

		@Override
		public long getPos() throws IOException {
			return original.getPos();
		}

		@Override
		public int read() throws IOException {
			if (closed) {
				throw new IOException("Stream closed");
			}
			return original.read();
		}

		@Override
		public void close() throws IOException {
			original.close();
			this.closed = true;
		}
	};
}
Example 11
Source File: RocksDBStateUploader.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
private StreamStateHandle uploadLocalFileToCheckpointFs(
	Path filePath,
	CheckpointStreamFactory checkpointStreamFactory,
	CloseableRegistry closeableRegistry) throws IOException {
	FSDataInputStream inputStream = null;
	CheckpointStreamFactory.CheckpointStateOutputStream outputStream = null;

	try {
		final byte[] buffer = new byte[READ_BUFFER_SIZE];

		FileSystem backupFileSystem = filePath.getFileSystem();
		inputStream = backupFileSystem.open(filePath);
		closeableRegistry.registerCloseable(inputStream);

		outputStream = checkpointStreamFactory
			.createCheckpointStateOutputStream(CheckpointedStateScope.SHARED);
		closeableRegistry.registerCloseable(outputStream);

		while (true) {
			int numBytes = inputStream.read(buffer);

			if (numBytes == -1) {
				break;
			}

			outputStream.write(buffer, 0, numBytes);
		}

		StreamStateHandle result = null;
		if (closeableRegistry.unregisterCloseable(outputStream)) {
			result = outputStream.closeAndGetHandle();
			outputStream = null;
		}
		return result;

	} finally {

		if (closeableRegistry.unregisterCloseable(inputStream)) {
			IOUtils.closeQuietly(inputStream);
		}

		if (closeableRegistry.unregisterCloseable(outputStream)) {
			IOUtils.closeQuietly(outputStream);
		}
	}
}
Example 12
Source File: RocksDBStateUploader.java    From flink with Apache License 2.0 4 votes vote down vote up 
private StreamStateHandle uploadLocalFileToCheckpointFs(
	Path filePath,
	CheckpointStreamFactory checkpointStreamFactory,
	CloseableRegistry closeableRegistry) throws IOException {
	FSDataInputStream inputStream = null;
	CheckpointStreamFactory.CheckpointStateOutputStream outputStream = null;

	try {
		final byte[] buffer = new byte[READ_BUFFER_SIZE];

		FileSystem backupFileSystem = filePath.getFileSystem();
		inputStream = backupFileSystem.open(filePath);
		closeableRegistry.registerCloseable(inputStream);

		outputStream = checkpointStreamFactory
			.createCheckpointStateOutputStream(CheckpointedStateScope.SHARED);
		closeableRegistry.registerCloseable(outputStream);

		while (true) {
			int numBytes = inputStream.read(buffer);

			if (numBytes == -1) {
				break;
			}

			outputStream.write(buffer, 0, numBytes);
		}

		StreamStateHandle result = null;
		if (closeableRegistry.unregisterCloseable(outputStream)) {
			result = outputStream.closeAndGetHandle();
			outputStream = null;
		}
		return result;

	} finally {

		if (closeableRegistry.unregisterCloseable(inputStream)) {
			IOUtils.closeQuietly(inputStream);
		}

		if (closeableRegistry.unregisterCloseable(outputStream)) {
			IOUtils.closeQuietly(outputStream);
		}
	}
}