Java Code Examples for org.apache.flink.runtime.io.network.buffer.Buffer#readableBytes()

@Test
public void testReadToBufferBuilder() throws IOException {
	byte[] data = generateData(100);
	BufferBuilder bufferBuilder = new BufferBuilder(HeapMemorySegment.FACTORY.allocateUnpooledSegment(data.length, null), FreeingBufferRecycler.INSTANCE);
	BufferConsumer bufferConsumer = bufferBuilder.createBufferConsumer();

	new ChannelStateSerializerImpl().readData(new ByteArrayInputStream(data), wrap(bufferBuilder), Integer.MAX_VALUE);

	assertFalse(bufferBuilder.isFinished());

	bufferBuilder.finish();
	Buffer buffer = bufferConsumer.build();

	assertEquals(data.length, buffer.readableBytes());
	byte[] actual = new byte[buffer.readableBytes()];
	buffer.asByteBuf().readBytes(actual);
	assertArrayEquals(data, actual);
}
 

private byte[] toByteArray(CloseableIterator<Buffer> unconsumed) {
	final List<Buffer> buffers = new ArrayList<>();
	try {
		unconsumed.forEachRemaining(buffers::add);
		byte[] result = new byte[buffers.stream().mapToInt(Buffer::readableBytes).sum()];
		int offset = 0;
		for (Buffer buffer : buffers) {
			int len = buffer.readableBytes();
			buffer.getNioBuffer(0, len).get(result, offset, len);
			offset += len;
		}
		return result;
	} finally {
		buffers.forEach(Buffer::recycleBuffer);
	}
}
 

private ReadResult internalReaderRecoveredState(ChannelStateReader reader, Buffer buffer) throws IOException {
	ReadResult result;
	try {
		result = reader.readInputData(channelInfo, buffer);
	} catch (Throwable t) {
		buffer.recycleBuffer();
		throw t;
	}
	if (buffer.readableBytes() > 0) {
		onRecoveredStateBuffer(buffer);
	} else {
		buffer.recycleBuffer();
	}
	return result;
}
 

@Nullable
BufferAndBacklog pollBuffer() {
	synchronized (buffers) {
		Buffer buffer = null;

		if (buffers.isEmpty()) {
			flushRequested = false;
		}

		while (!buffers.isEmpty()) {
			BufferConsumer bufferConsumer = buffers.peek();

			buffer = bufferConsumer.build();

			checkState(bufferConsumer.isFinished() || buffers.size() == 1,
				"When there are multiple buffers, an unfinished bufferConsumer can not be at the head of the buffers queue.");

			if (buffers.size() == 1) {
				// turn off flushRequested flag if we drained all of the available data
				flushRequested = false;
			}

			if (bufferConsumer.isFinished()) {
				buffers.pop().close();
				decreaseBuffersInBacklogUnsafe(bufferConsumer.isBuffer());
			}

			if (buffer.readableBytes() > 0) {
				break;
			}
			buffer.recycleBuffer();
			buffer = null;
			if (!bufferConsumer.isFinished()) {
				break;
			}
		}

		if (buffer == null) {
			return null;
		}

		updateStatistics(buffer);
		// Do not report last remaining buffer on buffers as available to read (assuming it's unfinished).
		// It will be reported for reading either on flush or when the number of buffers in the queue
		// will be 2 or more.
		return new BufferAndBacklog(
			buffer,
			isAvailableUnsafe(),
			getBuffersInBacklog(),
			nextBufferIsEventUnsafe());
	}
}
 

@Nullable
BufferAndBacklog pollBuffer() {
	synchronized (buffers) {
		Buffer buffer = null;

		if (buffers.isEmpty()) {
			flushRequested = false;
		}

		while (!buffers.isEmpty()) {
			BufferConsumer bufferConsumer = buffers.peek();

			buffer = bufferConsumer.build();

			checkState(bufferConsumer.isFinished() || buffers.size() == 1,
				"When there are multiple buffers, an unfinished bufferConsumer can not be at the head of the buffers queue.");

			if (buffers.size() == 1) {
				// turn off flushRequested flag if we drained all of the available data
				flushRequested = false;
			}

			if (bufferConsumer.isFinished()) {
				buffers.pop().close();
				decreaseBuffersInBacklogUnsafe(bufferConsumer.isBuffer());
			}

			if (buffer.readableBytes() > 0) {
				break;
			}
			buffer.recycleBuffer();
			buffer = null;
			if (!bufferConsumer.isFinished()) {
				break;
			}
		}

		if (buffer == null) {
			return null;
		}

		updateStatistics(buffer);
		// Do not report last remaining buffer on buffers as available to read (assuming it's unfinished).
		// It will be reported for reading either on flush or when the number of buffers in the queue
		// will be 2 or more.
		return new BufferAndBacklog(
			buffer,
			isAvailableUnsafe(),
			getBuffersInBacklog(),
			nextBufferIsEventUnsafe());
	}
}
 

@Nullable
BufferAndBacklog pollBuffer() {
	synchronized (buffers) {
		if (isBlockedByCheckpoint) {
			return null;
		}

		Buffer buffer = null;

		if (buffers.isEmpty()) {
			flushRequested = false;
		}

		while (!buffers.isEmpty()) {
			BufferConsumer bufferConsumer = buffers.peek();

			buffer = bufferConsumer.build();

			checkState(bufferConsumer.isFinished() || buffers.size() == 1,
				"When there are multiple buffers, an unfinished bufferConsumer can not be at the head of the buffers queue.");

			if (buffers.size() == 1) {
				// turn off flushRequested flag if we drained all of the available data
				flushRequested = false;
			}

			if (bufferConsumer.isFinished()) {
				buffers.pop().close();
				decreaseBuffersInBacklogUnsafe(bufferConsumer.isBuffer());
			}

			if (buffer.readableBytes() > 0) {
				break;
			}
			buffer.recycleBuffer();
			buffer = null;
			if (!bufferConsumer.isFinished()) {
				break;
			}
		}

		if (buffer == null) {
			return null;
		}

		if (buffer.getDataType().isBlockingUpstream()) {
			isBlockedByCheckpoint = true;
		}

		updateStatistics(buffer);
		// Do not report last remaining buffer on buffers as available to read (assuming it's unfinished).
		// It will be reported for reading either on flush or when the number of buffers in the queue
		// will be 2 or more.
		return new BufferAndBacklog(
			buffer,
			isDataAvailableUnsafe(),
			getBuffersInBacklog(),
			isEventAvailableUnsafe());
	}
}
 

/**
 * Whether the buffer can be compressed or not. Note that event is not compressed because it
 * is usually small and the size can become even larger after compression.
 */
protected boolean canBeCompressed(Buffer buffer) {
	return parent.bufferCompressor != null && buffer.isBuffer() && buffer.readableBytes() > 0;
}