Java Code Examples for org.apache.flink.runtime.io.network.buffer.Buffer#setSize()

The following examples show how to use org.apache.flink.runtime.io.network.buffer.Buffer#setSize() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: CheckpointBarrierAlignerTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up 
private static BufferOrEvent createBuffer(int channel) {
	final int size = sizeCounter++;
	byte[] bytes = new byte[size];
	RND.nextBytes(bytes);

	MemorySegment memory = MemorySegmentFactory.allocateUnpooledSegment(PAGE_SIZE);
	memory.put(0, bytes);

	Buffer buf = new NetworkBuffer(memory, FreeingBufferRecycler.INSTANCE);
	buf.setSize(size);

	// retain an additional time so it does not get disposed after being read by the input gate
	buf.retainBuffer();

	return new BufferOrEvent(buf, new InputChannelInfo(0, channel));
}
Example 2
Source File: BarrierBufferTestBase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
private static BufferOrEvent createBuffer(int channel, int pageSize) {
	final int size = sizeCounter++;
	byte[] bytes = new byte[size];
	RND.nextBytes(bytes);

	MemorySegment memory = MemorySegmentFactory.allocateUnpooledSegment(pageSize);
	memory.put(0, bytes);

	Buffer buf = new NetworkBuffer(memory, FreeingBufferRecycler.INSTANCE);
	buf.setSize(size);

	// retain an additional time so it does not get disposed after being read by the input gate
	buf.retainBuffer();

	return new BufferOrEvent(buf, channel);
}
Example 3
Source File: CheckpointBarrierAlignerTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up 
private static BufferOrEvent createBuffer(int channel) {
	final int size = sizeCounter++;
	byte[] bytes = new byte[size];
	RND.nextBytes(bytes);

	MemorySegment memory = MemorySegmentFactory.allocateUnpooledSegment(PAGE_SIZE);
	memory.put(0, bytes);

	Buffer buf = new NetworkBuffer(memory, FreeingBufferRecycler.INSTANCE);
	buf.setSize(size);

	// retain an additional time so it does not get disposed after being read by the input gate
	buf.retainBuffer();

	return new BufferOrEvent(buf, channel);
}
Example 4
Source File: BufferFileWriterReaderTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
static int fillBufferWithAscendingNumbers(Buffer buffer, int currentNumber, int size) {
	checkArgument(size % 4 == 0);

	MemorySegment segment = buffer.getMemorySegment();

	for (int i = 0; i < size; i += 4) {
		segment.putInt(i, currentNumber++);
	}
	buffer.setSize(size);

	return currentNumber;
}
Example 5
Source File: BufferFileWriterReaderTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
static int fillBufferWithAscendingNumbers(Buffer buffer, int currentNumber, int size) {
	checkArgument(size % 4 == 0);

	MemorySegment segment = buffer.getMemorySegment();

	for (int i = 0; i < size; i += 4) {
		segment.putInt(i, currentNumber++);
	}
	buffer.setSize(size);

	return currentNumber;
}
Example 6
Source File: SingleInputGateTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Tests that the compressed buffer will be decompressed after calling {@link SingleInputGate#getNext()}.
 */
@Test
public void testGetCompressedBuffer() throws Exception {
	int bufferSize = 1024;
	String compressionCodec = "LZ4";
	BufferCompressor compressor = new BufferCompressor(bufferSize, compressionCodec);
	BufferDecompressor decompressor = new BufferDecompressor(bufferSize, compressionCodec);

	try (SingleInputGate inputGate = new SingleInputGateBuilder().setBufferDecompressor(decompressor).build()) {
		TestInputChannel inputChannel = new TestInputChannel(inputGate, 0);

		MemorySegment segment = MemorySegmentFactory.allocateUnpooledSegment(bufferSize);
		for (int i = 0; i < bufferSize; i += 8) {
			segment.putLongLittleEndian(i, i);
		}
		Buffer uncompressedBuffer = new NetworkBuffer(segment, FreeingBufferRecycler.INSTANCE);
		uncompressedBuffer.setSize(bufferSize);
		Buffer compressedBuffer = compressor.compressToOriginalBuffer(uncompressedBuffer);
		assertTrue(compressedBuffer.isCompressed());

		inputChannel.read(compressedBuffer);
		inputGate.setInputChannels(inputChannel);
		inputGate.notifyChannelNonEmpty(inputChannel);

		Optional<BufferOrEvent> bufferOrEvent = inputGate.getNext();
		assertTrue(bufferOrEvent.isPresent());
		assertTrue(bufferOrEvent.get().isBuffer());
		ByteBuffer buffer = bufferOrEvent.get().getBuffer().getNioBufferReadable().order(ByteOrder.LITTLE_ENDIAN);
		for (int i = 0; i < bufferSize; i += 8) {
			assertEquals(i, buffer.getLong());
		}
	}
}
Example 7
Source File: CancelPartitionRequestTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Nullable
@Override
public BufferAndBacklog getNextBuffer() throws IOException {
	Buffer buffer = bufferProvider.requestBuffer();
	if (buffer != null) {
		buffer.setSize(buffer.getMaxCapacity()); // fake some data
		return new BufferAndBacklog(buffer, true, 0, false);
	} else {
		return null;
	}
}
Example 8
Source File: BufferStorageTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up 
public static BufferOrEvent generateRandomBuffer(int size, int channelIndex) {
	MemorySegment seg = MemorySegmentFactory.allocateUnpooledSegment(PAGE_SIZE);
	for (int i = 0; i < size; i++) {
		seg.put(i, (byte) i);
	}

	Buffer buf = new NetworkBuffer(seg, FreeingBufferRecycler.INSTANCE);
	buf.setSize(size);
	return new BufferOrEvent(buf, channelIndex);
}
Example 9
Source File: CheckpointBarrierAlignerAlignmentLimitTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
private static BufferOrEvent createBuffer(int channel, int size) {
	byte[] bytes = new byte[size];
	RND.nextBytes(bytes);

	MemorySegment memory = MemorySegmentFactory.allocateUnpooledSegment(PAGE_SIZE);
	memory.put(0, bytes);

	Buffer buf = new NetworkBuffer(memory, FreeingBufferRecycler.INSTANCE);
	buf.setSize(size);

	// retain an additional time so it does not get disposed after being read by the input gate
	buf.retainBuffer();

	return new BufferOrEvent(buf, channel);
}
Example 10
Source File: CancelPartitionRequestTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Nullable
@Override
public BufferAndBacklog getNextBuffer() throws IOException, InterruptedException {
	Buffer buffer = bufferProvider.requestBufferBlocking();
	buffer.setSize(buffer.getMaxCapacity()); // fake some data
	return new BufferAndBacklog(buffer, true, 0, false);
}
Example 11
Source File: BufferReaderWriterUtil.java    From flink with Apache License 2.0 5 votes vote down vote up 
static Buffer bufferFromMemorySegment(
		MemorySegment memorySegment,
		BufferRecycler memorySegmentRecycler,
		int size,
		boolean isEvent) {

	final Buffer buffer = new NetworkBuffer(memorySegment, memorySegmentRecycler);
	buffer.setSize(size);

	if (isEvent) {
		buffer.tagAsEvent();
	}

	return buffer;
}
Example 12
Source File: BufferBlockerTestBase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
public static BufferOrEvent generateRandomBuffer(int size, int channelIndex) {
	MemorySegment seg = MemorySegmentFactory.allocateUnpooledSegment(PAGE_SIZE);
	for (int i = 0; i < size; i++) {
		seg.put(i, (byte) i);
	}

	Buffer buf = new NetworkBuffer(seg, FreeingBufferRecycler.INSTANCE);
	buf.setSize(size);
	return new BufferOrEvent(buf, channelIndex);
}
Example 13
Source File: BarrierBufferAlignmentLimitTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
private static BufferOrEvent createBuffer(int channel, int size) {
	byte[] bytes = new byte[size];
	RND.nextBytes(bytes);

	MemorySegment memory = MemorySegmentFactory.allocateUnpooledSegment(PAGE_SIZE);
	memory.put(0, bytes);

	Buffer buf = new NetworkBuffer(memory, FreeingBufferRecycler.INSTANCE);
	buf.setSize(size);

	// retain an additional time so it does not get disposed after being read by the input gate
	buf.retainBuffer();

	return new BufferOrEvent(buf, channel);
}
Example 14
Source File: BufferFileWriterReaderTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
static int fillBufferWithAscendingNumbers(Buffer buffer, int currentNumber, int size) {
	checkArgument(size % 4 == 0);

	MemorySegment segment = buffer.getMemorySegment();

	for (int i = 0; i < size; i += 4) {
		segment.putInt(i, currentNumber++);
	}
	buffer.setSize(size);

	return currentNumber;
}
Example 15
Source File: CancelPartitionRequestTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Nullable
@Override
public BufferAndBacklog getNextBuffer() throws IOException, InterruptedException {
	Buffer buffer = bufferProvider.requestBufferBlocking();
	buffer.setSize(buffer.getMaxCapacity()); // fake some data
	return new BufferAndBacklog(buffer, true, 0, false);
}
Example 16
Source File: BufferSpiller.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Override
public BufferOrEvent getNext() throws IOException {
	if (buffer.remaining() < HEADER_LENGTH) {
		buffer.compact();

		while (buffer.position() < HEADER_LENGTH) {
			if (fileChannel.read(buffer) == -1) {
				if (buffer.position() == 0) {
					// no trailing data
					return null;
				} else {
					throw new IOException("Found trailing incomplete buffer or event");
				}
			}
		}

		buffer.flip();
	}

	final int channel = buffer.getInt();
	final int length = buffer.getInt();
	final boolean isBuffer = buffer.get() == 0;

	if (isBuffer) {
		// deserialize buffer
		if (length > pageSize) {
			throw new IOException(String.format(
					"Spilled buffer (%d bytes) is larger than page size of (%d bytes)", length, pageSize));
		}

		MemorySegment seg = MemorySegmentFactory.allocateUnpooledSegment(pageSize);

		int segPos = 0;
		int bytesRemaining = length;

		while (true) {
			int toCopy = Math.min(buffer.remaining(), bytesRemaining);
			if (toCopy > 0) {
				seg.put(segPos, buffer, toCopy);
				segPos += toCopy;
				bytesRemaining -= toCopy;
			}

			if (bytesRemaining == 0) {
				break;
			}
			else {
				buffer.clear();
				if (fileChannel.read(buffer) == -1) {
					throw new IOException("Found trailing incomplete buffer");
				}
				buffer.flip();
			}
		}

		Buffer buf = new NetworkBuffer(seg, FreeingBufferRecycler.INSTANCE);
		buf.setSize(length);

		return new BufferOrEvent(buf, channel, true);
	}
	else {
		// deserialize event
		if (length > buffer.capacity() - HEADER_LENGTH) {
			throw new IOException("Event is too large");
		}

		if (buffer.remaining() < length) {
			buffer.compact();

			while (buffer.position() < length) {
				if (fileChannel.read(buffer) == -1) {
					throw new IOException("Found trailing incomplete event");
				}
			}

			buffer.flip();
		}

		int oldLimit = buffer.limit();
		buffer.limit(buffer.position() + length);
		AbstractEvent evt = EventSerializer.fromSerializedEvent(buffer, getClass().getClassLoader());
		buffer.limit(oldLimit);

		return new BufferOrEvent(evt, channel, true, length);
	}
}
Example 17
Source File: BufferSpiller.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@Override
public BufferOrEvent getNext() throws IOException {
	if (buffer.remaining() < HEADER_LENGTH) {
		buffer.compact();

		while (buffer.position() < HEADER_LENGTH) {
			if (fileChannel.read(buffer) == -1) {
				if (buffer.position() == 0) {
					// no trailing data
					return null;
				} else {
					throw new IOException("Found trailing incomplete buffer or event");
				}
			}
		}

		buffer.flip();
	}

	final int channel = buffer.getInt();
	final int length = buffer.getInt();
	final boolean isBuffer = buffer.get() == 0;

	if (isBuffer) {
		// deserialize buffer
		if (length > pageSize) {
			throw new IOException(String.format(
					"Spilled buffer (%d bytes) is larger than page size of (%d bytes)", length, pageSize));
		}

		MemorySegment seg = MemorySegmentFactory.allocateUnpooledSegment(pageSize);

		int segPos = 0;
		int bytesRemaining = length;

		while (true) {
			int toCopy = Math.min(buffer.remaining(), bytesRemaining);
			if (toCopy > 0) {
				seg.put(segPos, buffer, toCopy);
				segPos += toCopy;
				bytesRemaining -= toCopy;
			}

			if (bytesRemaining == 0) {
				break;
			}
			else {
				buffer.clear();
				if (fileChannel.read(buffer) == -1) {
					throw new IOException("Found trailing incomplete buffer");
				}
				buffer.flip();
			}
		}

		Buffer buf = new NetworkBuffer(seg, FreeingBufferRecycler.INSTANCE);
		buf.setSize(length);

		return new BufferOrEvent(buf, channel);
	}
	else {
		// deserialize event
		if (length > buffer.capacity() - HEADER_LENGTH) {
			throw new IOException("Event is too large");
		}

		if (buffer.remaining() < length) {
			buffer.compact();

			while (buffer.position() < length) {
				if (fileChannel.read(buffer) == -1) {
					throw new IOException("Found trailing incomplete event");
				}
			}

			buffer.flip();
		}

		int oldLimit = buffer.limit();
		buffer.limit(buffer.position() + length);
		AbstractEvent evt = EventSerializer.fromSerializedEvent(buffer, getClass().getClassLoader());
		buffer.limit(oldLimit);

		return new BufferOrEvent(evt, channel);
	}
}
Example 18
Source File: EventSerializer.java    From Flink-CEPplus with Apache License 2.0 3 votes vote down vote up 
public static Buffer toBuffer(AbstractEvent event) throws IOException {
	final ByteBuffer serializedEvent = EventSerializer.toSerializedEvent(event);

	MemorySegment data = MemorySegmentFactory.wrap(serializedEvent.array());

	final Buffer buffer = new NetworkBuffer(data, FreeingBufferRecycler.INSTANCE, false);
	buffer.setSize(serializedEvent.remaining());

	return buffer;
}
Example 19
Source File: EventSerializer.java    From flink with Apache License 2.0 3 votes vote down vote up 
public static Buffer toBuffer(AbstractEvent event) throws IOException {
	final ByteBuffer serializedEvent = EventSerializer.toSerializedEvent(event);

	MemorySegment data = MemorySegmentFactory.wrap(serializedEvent.array());

	final Buffer buffer = new NetworkBuffer(data, FreeingBufferRecycler.INSTANCE, false);
	buffer.setSize(serializedEvent.remaining());

	return buffer;
}
Example 20
Source File: EventSerializer.java    From flink with Apache License 2.0 3 votes vote down vote up 
public static Buffer toBuffer(AbstractEvent event) throws IOException {
	final ByteBuffer serializedEvent = EventSerializer.toSerializedEvent(event);

	MemorySegment data = MemorySegmentFactory.wrap(serializedEvent.array());

	final Buffer buffer = new NetworkBuffer(data, FreeingBufferRecycler.INSTANCE, Buffer.DataType.getDataType(event));
	buffer.setSize(serializedEvent.remaining());

	return buffer;
}