Java Code Examples for org.apache.flink.shaded.netty4.io.netty.buffer.ByteBuf#writeInt()

/**
 * Allocates a new buffer and adds some header information for the frame decoder.
 *
 * <p>If the <tt>contentLength</tt> is unknown, you must write the actual length after adding
 * the contents as an integer to position <tt>0</tt>!
 *
 * @param allocator
 * 		byte buffer allocator to use
 * @param id
 * 		{@link NettyMessage} subclass ID
 * @param messageHeaderLength
 * 		additional header length that should be part of the allocated buffer and is written
 * 		outside of this method
 * @param contentLength
 * 		content length (or <tt>-1</tt> if unknown)
 * @param allocateForContent
 * 		whether to make room for the actual content in the buffer (<tt>true</tt>) or whether to
 * 		only return a buffer with the header information (<tt>false</tt>)
 *
 * @return a newly allocated direct buffer with header data written for {@link
 * NettyMessageEncoder}
 */
private static ByteBuf allocateBuffer(
		ByteBufAllocator allocator,
		byte id,
		int messageHeaderLength,
		int contentLength,
		boolean allocateForContent) {
	checkArgument(contentLength <= Integer.MAX_VALUE - FRAME_HEADER_LENGTH);

	final ByteBuf buffer;
	if (!allocateForContent) {
		buffer = allocator.directBuffer(FRAME_HEADER_LENGTH + messageHeaderLength);
	} else if (contentLength != -1) {
		buffer = allocator.directBuffer(FRAME_HEADER_LENGTH + messageHeaderLength + contentLength);
	} else {
		// content length unknown -> start with the default initial size (rather than FRAME_HEADER_LENGTH only):
		buffer = allocator.directBuffer();
	}
	buffer.writeInt(FRAME_HEADER_LENGTH + messageHeaderLength + contentLength); // may be updated later, e.g. if contentLength == -1
	buffer.writeInt(MAGIC_NUMBER);
	buffer.writeByte(id);

	return buffer;
}
 

/**
 * Serializes the failure message sent to the
 * {@link org.apache.flink.queryablestate.network.Client} in case of
 * server related errors.
 *
 * @param alloc			The {@link ByteBufAllocator} used to allocate the buffer to serialize the message into.
 * @param cause			The exception thrown at the server.
 * @return		The failure message.
 */
public static ByteBuf serializeServerFailure(
		final ByteBufAllocator alloc,
		final Throwable cause) throws IOException {

	final ByteBuf buf = alloc.ioBuffer();

	// Frame length is set at end
	buf.writeInt(0);
	writeHeader(buf, MessageType.SERVER_FAILURE);

	try (ByteBufOutputStream bbos = new ByteBufOutputStream(buf);
			ObjectOutput out = new ObjectOutputStream(bbos)) {
		out.writeObject(cause);
	}

	// Set frame length
	int frameLength = buf.readableBytes() - Integer.BYTES;
	buf.setInt(0, frameLength);
	return buf;
}
 

/**
 * Serializes the exception containing the failure message sent to the
 * {@link org.apache.flink.queryablestate.network.Client} in case of
 * protocol related errors.
 *
 * @param alloc			The {@link ByteBufAllocator} used to allocate the buffer to serialize the message into.
 * @param requestId		The id of the request to which the message refers to.
 * @param cause			The exception thrown at the server.
 * @return A {@link ByteBuf} containing the serialized message.
 */
public static ByteBuf serializeRequestFailure(
		final ByteBufAllocator alloc,
		final long requestId,
		final Throwable cause) throws IOException {

	final ByteBuf buf = alloc.ioBuffer();

	// Frame length is set at the end
	buf.writeInt(0);
	writeHeader(buf, MessageType.REQUEST_FAILURE);
	buf.writeLong(requestId);

	try (ByteBufOutputStream bbos = new ByteBufOutputStream(buf);
			ObjectOutput out = new ObjectOutputStream(bbos)) {
		out.writeObject(cause);
	}

	// Set frame length
	int frameLength = buf.readableBytes() - Integer.BYTES;
	buf.setInt(0, frameLength);
	return buf;
}
 

@Override
ByteBuf write(ByteBufAllocator allocator) throws IOException {
	ByteBuf result = null;

	try {
		result = allocateBuffer(allocator, ID, 16 + 16 + 4 + 16);

		partitionId.getPartitionId().writeTo(result);
		partitionId.getProducerId().writeTo(result);
		result.writeInt(credit);
		receiverId.writeTo(result);

		return result;
	}
	catch (Throwable t) {
		if (result != null) {
			result.release();
		}

		throw new IOException(t);
	}
}
 

@Override
ByteBuf write(ByteBufAllocator allocator) throws IOException {
	ByteBuf result = null;

	try {
		result = allocateBuffer(allocator, ID, 16 + 16 + 4 + 16 + 4);

		partitionId.getPartitionId().writeTo(result);
		partitionId.getProducerId().writeTo(result);
		result.writeInt(queueIndex);
		receiverId.writeTo(result);
		result.writeInt(credit);

		return result;
	}
	catch (Throwable t) {
		if (result != null) {
			result.release();
		}

		throw new IOException(t);
	}
}
 

/**
 * Allocates a new buffer and adds some header information for the frame decoder.
 *
 * <p>If the <tt>contentLength</tt> is unknown, you must write the actual length after adding
 * the contents as an integer to position <tt>0</tt>!
 *
 * @param allocator
 * 		byte buffer allocator to use
 * @param id
 * 		{@link NettyMessage} subclass ID
 * @param messageHeaderLength
 * 		additional header length that should be part of the allocated buffer and is written
 * 		outside of this method
 * @param contentLength
 * 		content length (or <tt>-1</tt> if unknown)
 * @param allocateForContent
 * 		whether to make room for the actual content in the buffer (<tt>true</tt>) or whether to
 * 		only return a buffer with the header information (<tt>false</tt>)
 *
 * @return a newly allocated direct buffer with header data written for {@link
 * NettyMessageDecoder}
 */
private static ByteBuf allocateBuffer(
		ByteBufAllocator allocator,
		byte id,
		int messageHeaderLength,
		int contentLength,
		boolean allocateForContent) {
	checkArgument(contentLength <= Integer.MAX_VALUE - FRAME_HEADER_LENGTH);

	final ByteBuf buffer;
	if (!allocateForContent) {
		buffer = allocator.directBuffer(FRAME_HEADER_LENGTH + messageHeaderLength);
	} else if (contentLength != -1) {
		buffer = allocator.directBuffer(FRAME_HEADER_LENGTH + messageHeaderLength + contentLength);
	} else {
		// content length unknown -> start with the default initial size (rather than FRAME_HEADER_LENGTH only):
		buffer = allocator.directBuffer();
	}
	buffer.writeInt(FRAME_HEADER_LENGTH + messageHeaderLength + contentLength); // may be updated later, e.g. if contentLength == -1
	buffer.writeInt(MAGIC_NUMBER);
	buffer.writeByte(id);

	return buffer;
}
 

/**
 * Serializes the failure message sent to the
 * {@link org.apache.flink.queryablestate.network.Client} in case of
 * server related errors.
 *
 * @param alloc			The {@link ByteBufAllocator} used to allocate the buffer to serialize the message into.
 * @param cause			The exception thrown at the server.
 * @return		The failure message.
 */
public static ByteBuf serializeServerFailure(
		final ByteBufAllocator alloc,
		final Throwable cause) throws IOException {

	final ByteBuf buf = alloc.ioBuffer();

	// Frame length is set at end
	buf.writeInt(0);
	writeHeader(buf, MessageType.SERVER_FAILURE);

	try (ByteBufOutputStream bbos = new ByteBufOutputStream(buf);
			ObjectOutput out = new ObjectOutputStream(bbos)) {
		out.writeObject(cause);
	}

	// Set frame length
	int frameLength = buf.readableBytes() - Integer.BYTES;
	buf.setInt(0, frameLength);
	return buf;
}
 

/**
 * Serializes the exception containing the failure message sent to the
 * {@link org.apache.flink.queryablestate.network.Client} in case of
 * protocol related errors.
 *
 * @param alloc			The {@link ByteBufAllocator} used to allocate the buffer to serialize the message into.
 * @param requestId		The id of the request to which the message refers to.
 * @param cause			The exception thrown at the server.
 * @return A {@link ByteBuf} containing the serialized message.
 */
public static ByteBuf serializeRequestFailure(
		final ByteBufAllocator alloc,
		final long requestId,
		final Throwable cause) throws IOException {

	final ByteBuf buf = alloc.ioBuffer();

	// Frame length is set at the end
	buf.writeInt(0);
	writeHeader(buf, MessageType.REQUEST_FAILURE);
	buf.writeLong(requestId);

	try (ByteBufOutputStream bbos = new ByteBufOutputStream(buf);
			ObjectOutput out = new ObjectOutputStream(bbos)) {
		out.writeObject(cause);
	}

	// Set frame length
	int frameLength = buf.readableBytes() - Integer.BYTES;
	buf.setInt(0, frameLength);
	return buf;
}
 

/**
 * Helper for serializing the messages.
 *
 * @param alloc			The {@link ByteBufAllocator} used to allocate the buffer to serialize the message into.
 * @param requestId		The id of the request to which the message refers to.
 * @param messageType	The {@link MessageType type of the message}.
 * @param payload		The serialized version of the message.
 * @return A {@link ByteBuf} containing the serialized message.
 */
private static ByteBuf writePayload(
		final ByteBufAllocator alloc,
		final long requestId,
		final MessageType messageType,
		final byte[] payload) {

	final int frameLength = HEADER_LENGTH + REQUEST_ID_SIZE + payload.length;
	final ByteBuf buf = alloc.ioBuffer(frameLength + Integer.BYTES);

	buf.writeInt(frameLength);
	writeHeader(buf, messageType);
	buf.writeLong(requestId);
	buf.writeBytes(payload);
	return buf;
}
 

@Override
ByteBuf write(ByteBufAllocator allocator) throws IOException {
	ByteBuf result = null;

	try {
		result = allocateBuffer(allocator, ID, 4 + 16);
		result.writeInt(credit);
		receiverId.writeTo(result);

		return result;
	}
	catch (Throwable t) {
		if (result != null) {
			result.release();
		}

		throw new IOException(t);
	}
}
 

/**
 * Helper for serializing the messages.
 *
 * @param alloc			The {@link ByteBufAllocator} used to allocate the buffer to serialize the message into.
 * @param requestId		The id of the request to which the message refers to.
 * @param messageType	The {@link MessageType type of the message}.
 * @param payload		The serialized version of the message.
 * @return A {@link ByteBuf} containing the serialized message.
 */
private static ByteBuf writePayload(
		final ByteBufAllocator alloc,
		final long requestId,
		final MessageType messageType,
		final byte[] payload) {

	final int frameLength = HEADER_LENGTH + REQUEST_ID_SIZE + payload.length;
	final ByteBuf buf = alloc.ioBuffer(frameLength + Integer.BYTES);

	buf.writeInt(frameLength);
	writeHeader(buf, messageType);
	buf.writeLong(requestId);
	buf.writeBytes(payload);
	return buf;
}
 

/**
 * Tests that incoming buffer instances are recycled.
 */
@Test
public void testIncomingBufferIsRecycled() throws Exception {
	KvStateRegistry registry = new KvStateRegistry();
	AtomicKvStateRequestStats stats = new AtomicKvStateRequestStats();

	MessageSerializer<KvStateInternalRequest, KvStateResponse> serializer =
			new MessageSerializer<>(new KvStateInternalRequest.KvStateInternalRequestDeserializer(), new KvStateResponse.KvStateResponseDeserializer());

	KvStateServerHandler handler = new KvStateServerHandler(testServer, registry, serializer, stats);
	EmbeddedChannel channel = new EmbeddedChannel(getFrameDecoder(), handler);

	KvStateInternalRequest request = new KvStateInternalRequest(new KvStateID(), new byte[0]);
	ByteBuf serRequest = MessageSerializer.serializeRequest(channel.alloc(), 282872L, request);

	assertEquals(1L, serRequest.refCnt());

	// Write regular request
	channel.writeInbound(serRequest);
	assertEquals("Buffer not recycled", 0L, serRequest.refCnt());

	// Write unexpected msg
	ByteBuf unexpected = channel.alloc().buffer(8);
	unexpected.writeInt(4);
	unexpected.writeInt(4);

	assertEquals(1L, unexpected.refCnt());

	channel.writeInbound(unexpected);
	assertEquals("Buffer not recycled", 0L, unexpected.refCnt());
}
 

@Override
ByteBuf write(ByteBufAllocator allocator) throws IOException {
	ByteBuf headerBuf = null;
	try {
		// in order to forward the buffer to netty, it needs an allocator set
		buffer.setAllocator(allocator);

		// only allocate header buffer - we will combine it with the data buffer below
		headerBuf = allocateBuffer(allocator, ID, MESSAGE_HEADER_LENGTH, bufferSize, false);

		receiverId.writeTo(headerBuf);
		headerBuf.writeInt(sequenceNumber);
		headerBuf.writeInt(backlog);
		headerBuf.writeByte(dataType.ordinal());
		headerBuf.writeBoolean(isCompressed);
		headerBuf.writeInt(buffer.readableBytes());

		CompositeByteBuf composityBuf = allocator.compositeDirectBuffer();
		composityBuf.addComponent(headerBuf);
		composityBuf.addComponent(buffer.asByteBuf());
		// update writer index since we have data written to the components:
		composityBuf.writerIndex(headerBuf.writerIndex() + buffer.asByteBuf().writerIndex());
		return composityBuf;
	}
	catch (Throwable t) {
		if (headerBuf != null) {
			headerBuf.release();
		}
		buffer.recycleBuffer();

		ExceptionUtils.rethrowIOException(t);
		return null; // silence the compiler
	}
}
 

@Override
ByteBuf write(ByteBufAllocator allocator) throws IOException {
	ByteBuf result = null;

	try {
		// TODO Directly serialize to Netty's buffer
		ByteBuffer serializedEvent = EventSerializer.toSerializedEvent(event);

		result = allocateBuffer(allocator, ID, 4 + serializedEvent.remaining() + 20 + 16 + 16);

		result.writeInt(serializedEvent.remaining());
		result.writeBytes(serializedEvent);

		partitionId.getPartitionId().writeTo(result);
		partitionId.getProducerId().writeTo(result);

		receiverId.writeTo(result);

		return result;
	}
	catch (Throwable t) {
		if (result != null) {
			result.release();
		}

		throw new IOException(t);
	}
}
 

@Override
ByteBuf write(ByteBufAllocator allocator) throws IOException {
	ByteBuf result = null;

	try {
		// TODO Directly serialize to Netty's buffer
		ByteBuffer serializedEvent = EventSerializer.toSerializedEvent(event);

		result = allocateBuffer(allocator, ID, 4 + serializedEvent.remaining() + 16 + 16 + 16);

		result.writeInt(serializedEvent.remaining());
		result.writeBytes(serializedEvent);

		partitionId.getPartitionId().writeTo(result);
		partitionId.getProducerId().writeTo(result);

		receiverId.writeTo(result);

		return result;
	}
	catch (Throwable t) {
		if (result != null) {
			result.release();
		}

		throw new IOException(t);
	}
}
 

@Test
public void testLittleEndianWithExpand() {
    ByteBuf buffer = newBuffer(0).order(LITTLE_ENDIAN);
    buffer.writeInt(0x12345678);
    assertEquals("78563412", ByteBufUtil.hexDump(buffer));
    buffer.release();
}
 

@Override
ByteBuf write(ByteBufAllocator allocator) throws IOException {
	// receiver ID (16), sequence number (4), backlog (4), isBuffer (1), buffer size (4)
	final int messageHeaderLength = 16 + 4 + 4 + 1 + 4;

	ByteBuf headerBuf = null;
	try {
		if (buffer instanceof Buffer) {
			// in order to forward the buffer to netty, it needs an allocator set
			((Buffer) buffer).setAllocator(allocator);
		}

		// only allocate header buffer - we will combine it with the data buffer below
		headerBuf = allocateBuffer(allocator, ID, messageHeaderLength, buffer.readableBytes(), false);

		receiverId.writeTo(headerBuf);
		headerBuf.writeInt(sequenceNumber);
		headerBuf.writeInt(backlog);
		headerBuf.writeBoolean(isBuffer);
		headerBuf.writeInt(buffer.readableBytes());

		CompositeByteBuf composityBuf = allocator.compositeDirectBuffer();
		composityBuf.addComponent(headerBuf);
		composityBuf.addComponent(buffer);
		// update writer index since we have data written to the components:
		composityBuf.writerIndex(headerBuf.writerIndex() + buffer.writerIndex());
		return composityBuf;
	}
	catch (Throwable t) {
		if (headerBuf != null) {
			headerBuf.release();
		}
		buffer.release();

		ExceptionUtils.rethrowIOException(t);
		return null; // silence the compiler
	}
}
 

public void writeTo(ByteBuf buf) {
	intermediateDataSetID.writeTo(buf);
	buf.writeInt(partitionNum);
}
 

/**
 * Helper for serializing the header.
 *
 * @param buf         The {@link ByteBuf} to serialize the header into.
 * @param messageType The {@link MessageType} of the message this header refers to.
 */
private static void writeHeader(final ByteBuf buf, final MessageType messageType) {
	buf.writeInt(VERSION);
	buf.writeInt(messageType.ordinal());
}
 

/**
 * Helper for serializing the header.
 *
 * @param buf         The {@link ByteBuf} to serialize the header into.
 * @param messageType The {@link MessageType} of the message this header refers to.
 */
private static void writeHeader(final ByteBuf buf, final MessageType messageType) {
	buf.writeInt(VERSION);
	buf.writeInt(messageType.ordinal());
}