Java Code Examples for org.apache.flink.runtime.io.network.buffer.Buffer#getSize()

@Override
public void setNextBuffer(Buffer buffer) throws IOException {
	currentBuffer = buffer;

	int offset = buffer.getMemorySegmentOffset();
	MemorySegment segment = buffer.getMemorySegment();
	int numBytes = buffer.getSize();

	// check if some spanning record deserialization is pending
	if (this.spanningWrapper.getNumGatheredBytes() > 0) {
		this.spanningWrapper.addNextChunkFromMemorySegment(segment, offset, numBytes);
	}
	else {
		this.nonSpanningWrapper.initializeFromMemorySegment(segment, offset, numBytes + offset);
	}
}
 

private static void readInts(BoundedData.Reader reader, int numBuffersExpected, int numInts) throws IOException {
	Buffer b;
	int nextValue = 0;
	int numBuffers = 0;

	while ((b = reader.nextBuffer()) != null) {
		final int numIntsInBuffer = b.getSize() / 4;
		BufferBuilderTestUtils.validateBufferWithAscendingInts(b, numIntsInBuffer, nextValue);
		nextValue += numIntsInBuffer;
		numBuffers++;

		b.recycleBuffer();
	}

	assertEquals(numBuffersExpected, numBuffers);
	assertThat(nextValue, Matchers.greaterThanOrEqualTo(numInts));
}
 

private static void readInts(BoundedData.Reader reader, int numBuffersExpected, int numInts) throws IOException {
	Buffer b;
	int nextValue = 0;
	int numBuffers = 0;

	while ((b = reader.nextBuffer()) != null) {
		final int numIntsInBuffer = b.getSize() / 4;
		if (compressionEnabled && b.isCompressed()) {
			Buffer decompressedBuffer = DECOMPRESSOR.decompressToIntermediateBuffer(b);
			BufferBuilderTestUtils.validateBufferWithAscendingInts(decompressedBuffer, numIntsInBuffer, nextValue);
		} else {
			BufferBuilderTestUtils.validateBufferWithAscendingInts(b, numIntsInBuffer, nextValue);
		}
		nextValue += numIntsInBuffer;
		numBuffers++;

		b.recycleBuffer();
	}

	assertEquals(numBuffersExpected, numBuffers);
	assertThat(nextValue, Matchers.greaterThanOrEqualTo(numInts));
}
 

static long writeToByteChannelIfBelowSize(
		FileChannel channel,
		Buffer buffer,
		ByteBuffer[] arrayWithHeaderBuffer,
		long bytesLeft) throws IOException {

	if (bytesLeft >= HEADER_LENGTH + buffer.getSize()) {
		return writeToByteChannel(channel, buffer, arrayWithHeaderBuffer);
	}

	return -1L;
}
 

static boolean writeBuffer(Buffer buffer, ByteBuffer memory) {
	final int bufferSize = buffer.getSize();

	if (memory.remaining() < bufferSize + HEADER_LENGTH) {
		return false;
	}

	memory.putShort(buffer.isBuffer() ? HEADER_VALUE_IS_BUFFER : HEADER_VALUE_IS_EVENT);
	memory.putShort(buffer.isCompressed() ? BUFFER_IS_COMPRESSED : BUFFER_IS_NOT_COMPRESSED);
	memory.putInt(bufferSize);
	memory.put(buffer.getNioBufferReadable());
	return true;
}
 

public BufferOrEvent(Buffer buffer, InputChannelInfo channelInfo, boolean moreAvailable) {
	this.buffer = checkNotNull(buffer);
	this.event = null;
	this.channelInfo = channelInfo;
	this.moreAvailable = moreAvailable;
	this.size = buffer.getSize();
}
 

private BufferOrEvent transformEvent(
		Buffer buffer,
		boolean moreAvailable,
		InputChannel currentChannel) throws IOException, InterruptedException {
	final AbstractEvent event;
	try {
		event = EventSerializer.fromBuffer(buffer, getClass().getClassLoader());
	} finally {
		buffer.recycleBuffer();
	}

	if (event.getClass() == EndOfPartitionEvent.class) {
		channelsWithEndOfPartitionEvents.set(currentChannel.getChannelIndex());

		if (channelsWithEndOfPartitionEvents.cardinality() == numberOfInputChannels) {
			// Because of race condition between:
			// 1. releasing inputChannelsWithData lock in this method and reaching this place
			// 2. empty data notification that re-enqueues a channel
			// we can end up with moreAvailable flag set to true, while we expect no more data.
			checkState(!moreAvailable || !pollNext().isPresent());
			moreAvailable = false;
			hasReceivedAllEndOfPartitionEvents = true;
			markAvailable();
		}

		currentChannel.releaseAllResources();
	}

	return new BufferOrEvent(event, currentChannel.getChannelInfo(), moreAvailable, buffer.getSize());
}
 

static int verifyBufferFilledWithAscendingNumbers(Buffer buffer, int currentNumber) {
	MemorySegment segment = buffer.getMemorySegment();

	int size = buffer.getSize();

	for (int i = 0; i < size; i += 4) {
		if (segment.getInt(i) != currentNumber++) {
			throw new IllegalStateException("Read unexpected number from buffer.");
		}
	}

	return currentNumber;
}
 

@Override
public void setNextBuffer(Buffer buffer) throws IOException {
	currentBuffer = buffer;

	int offset = buffer.getMemorySegmentOffset();
	MemorySegment segment = buffer.getMemorySegment();
	int numBytes = buffer.getSize();

	// check if some spanning record deserialization is pending
	if (spanningWrapper.getNumGatheredBytes() > 0) {
		spanningWrapper.addNextChunkFromMemorySegment(segment, offset, numBytes);
	} else {
		nonSpanningWrapper.initializeFromMemorySegment(segment, offset, numBytes + offset);
	}
}
 

static int verifyBufferFilledWithAscendingNumbers(Buffer buffer, int currentNumber) {
	MemorySegment segment = buffer.getMemorySegment();

	int size = buffer.getSize();

	for (int i = 0; i < size; i += 4) {
		if (segment.getInt(i) != currentNumber++) {
			throw new IllegalStateException("Read unexpected number from buffer.");
		}
	}

	return currentNumber;
}
 

static boolean writeBuffer(Buffer buffer, ByteBuffer memory) {
	final int bufferSize = buffer.getSize();

	if (memory.remaining() < bufferSize + HEADER_LENGTH) {
		return false;
	}

	memory.putInt(buffer.isBuffer() ? HEADER_VALUE_IS_BUFFER : HEADER_VALUE_IS_EVENT);
	memory.putInt(bufferSize);
	memory.put(buffer.getNioBufferReadable());
	return true;
}
 

public BufferOrEvent(Buffer buffer, int channelIndex, boolean moreAvailable) {
	this.buffer = checkNotNull(buffer);
	this.event = null;
	this.channelIndex = channelIndex;
	this.moreAvailable = moreAvailable;
	this.size = buffer.getSize();
}
 

private BufferOrEvent transformToBufferOrEvent(
		Buffer buffer,
		boolean moreAvailable,
		InputChannel currentChannel) throws IOException, InterruptedException {
	if (buffer.isBuffer()) {
		return new BufferOrEvent(buffer, currentChannel.getChannelIndex(), moreAvailable);
	}
	else {
		final AbstractEvent event;
		try {
			event = EventSerializer.fromBuffer(buffer, getClass().getClassLoader());
		}
		finally {
			buffer.recycleBuffer();
		}

		if (event.getClass() == EndOfPartitionEvent.class) {
			channelsWithEndOfPartitionEvents.set(currentChannel.getChannelIndex());

			if (channelsWithEndOfPartitionEvents.cardinality() == numberOfInputChannels) {
				// Because of race condition between:
				// 1. releasing inputChannelsWithData lock in this method and reaching this place
				// 2. empty data notification that re-enqueues a channel
				// we can end up with moreAvailable flag set to true, while we expect no more data.
				checkState(!moreAvailable || !pollNext().isPresent());
				moreAvailable = false;
				hasReceivedAllEndOfPartitionEvents = true;
				markAvailable();
			}

			currentChannel.notifySubpartitionConsumed();
			currentChannel.releaseAllResources();
		}

		return new BufferOrEvent(event, currentChannel.getChannelIndex(), moreAvailable, buffer.getSize());
	}
}
 

static int verifyBufferFilledWithAscendingNumbers(Buffer buffer, int currentNumber) {
	MemorySegment segment = buffer.getMemorySegment();

	int size = buffer.getSize();

	for (int i = 0; i < size; i += 4) {
		if (segment.getInt(i) != currentNumber++) {
			throw new IllegalStateException("Read unexpected number from buffer.");
		}
	}

	return currentNumber;
}
 

static long writeToByteChannelIfBelowSize(
		FileChannel channel,
		Buffer buffer,
		ByteBuffer[] arrayWithHeaderBuffer,
		long bytesLeft) throws IOException {

	if (bytesLeft >= HEADER_LENGTH + buffer.getSize()) {
		return writeToByteChannel(channel, buffer, arrayWithHeaderBuffer);
	}

	return -1L;
}
 

@VisibleForTesting
long spillFinishedBufferConsumers(boolean forceFinishRemainingBuffers) throws IOException {
	long spilledBytes = 0;

	while (!buffers.isEmpty()) {
		BufferConsumer bufferConsumer = buffers.getFirst();
		Buffer buffer = bufferConsumer.build();
		updateStatistics(buffer);
		int bufferSize = buffer.getSize();
		spilledBytes += bufferSize;

		// NOTE we may be in the process of finishing the subpartition where any buffer should
		// be treated as if it was finished!
		if (bufferConsumer.isFinished() || forceFinishRemainingBuffers) {
			if (bufferSize > 0) {
				spillWriter.writeBlock(buffer);
			} else {
				// If we skip a buffer for the spill writer, we need to adapt the backlog accordingly
				decreaseBuffersInBacklog(buffer);
				buffer.recycleBuffer();
			}
			bufferConsumer.close();
			buffers.poll();
		} else {
			// If there is already data, we need to spill it anyway, since we do not get this
			// slice from the buffer consumer again during the next build.
			// BEWARE: by doing so, we increase the actual number of buffers in the spill writer!
			if (bufferSize > 0) {
				spillWriter.writeBlock(buffer);
				increaseBuffersInBacklog(bufferConsumer);
			} else {
				buffer.recycleBuffer();
			}

			return spilledBytes;
		}
	}
	return spilledBytes;
}
 

private void updateStatistics(Buffer buffer) {
	totalNumberOfBytes += buffer.getSize();
}
 

private void updateStatistics(Buffer buffer) {
	totalNumberOfBytes += buffer.getSize();
}
 

protected void updateStatistics(Buffer buffer) {
	totalNumberOfBytes += buffer.getSize();
}
 

int releaseMemory() throws IOException {
	synchronized (buffers) {
		if (spilledView != null || nextBuffer == null) {
			// Already spilled or nothing in-memory
			return 0;
		} else {
			// We don't touch next buffer, because a notification has
			// already been sent for it. Only when it is consumed, will
			// it be recycled.

			// Create the spill writer and write all buffers to disk
			BufferFileWriter spillWriter = ioManager.createBufferFileWriter(ioManager.createChannel());

			long spilledBytes = 0;

			int numBuffers = buffers.size();
			for (int i = 0; i < numBuffers; i++) {
				try (BufferConsumer bufferConsumer = buffers.remove()) {
					Buffer buffer = bufferConsumer.build();
					checkState(bufferConsumer.isFinished(), "BufferConsumer must be finished before " +
						"spilling. Otherwise we would not be able to simply remove it from the queue. This should " +
						"be guaranteed by creating ResultSubpartitionView only once Subpartition isFinished.");
					parent.updateStatistics(buffer);
					spilledBytes += buffer.getSize();
					spillWriter.writeBlock(buffer);
				}
			}

			spilledView = new SpilledSubpartitionView(
				parent,
				memorySegmentSize,
				spillWriter,
				numBuffers,
				listener);

			LOG.debug("Spilling {} bytes for sub partition {} of {}.",
				spilledBytes,
				parent.index,
				parent.parent.getPartitionId());

			return numBuffers;
		}
	}
}