Java Code Examples for org.apache.flink.runtime.io.network.api.serialization.EventSerializer#fromSerializedEvent()

The following examples show how to use org.apache.flink.runtime.io.network.api.serialization.EventSerializer#fromSerializedEvent() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: NettyMessage.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
static TaskEventRequest readFrom(ByteBuf buffer, ClassLoader classLoader) throws IOException {
	// directly deserialize fromNetty's buffer
	int length = buffer.readInt();
	ByteBuffer serializedEvent = buffer.nioBuffer(buffer.readerIndex(), length);
	// assume this event's content is read from the ByteBuf (positions are not shared!)
	buffer.readerIndex(buffer.readerIndex() + length);

	TaskEvent event =
		(TaskEvent) EventSerializer.fromSerializedEvent(serializedEvent, classLoader);

	ResultPartitionID partitionId =
		new ResultPartitionID(
			IntermediateResultPartitionID.fromByteBuf(buffer),
			ExecutionAttemptID.fromByteBuf(buffer));

	InputChannelID receiverId = InputChannelID.fromByteBuf(buffer);

	return new TaskEventRequest(event, partitionId, receiverId);
}
Example 2
Source File: NettyMessage.java    From flink with Apache License 2.0 6 votes vote down vote up 
static TaskEventRequest readFrom(ByteBuf buffer, ClassLoader classLoader) throws IOException {
	// directly deserialize fromNetty's buffer
	int length = buffer.readInt();
	ByteBuffer serializedEvent = buffer.nioBuffer(buffer.readerIndex(), length);
	// assume this event's content is read from the ByteBuf (positions are not shared!)
	buffer.readerIndex(buffer.readerIndex() + length);

	TaskEvent event =
		(TaskEvent) EventSerializer.fromSerializedEvent(serializedEvent, classLoader);

	ResultPartitionID partitionId =
		new ResultPartitionID(
			IntermediateResultPartitionID.fromByteBuf(buffer),
			ExecutionAttemptID.fromByteBuf(buffer));

	InputChannelID receiverId = InputChannelID.fromByteBuf(buffer);

	return new TaskEventRequest(event, partitionId, receiverId);
}
Example 3
Source File: NettyMessage.java    From flink with Apache License 2.0 6 votes vote down vote up 
static TaskEventRequest readFrom(ByteBuf buffer, ClassLoader classLoader) throws IOException {
	// directly deserialize fromNetty's buffer
	int length = buffer.readInt();
	ByteBuffer serializedEvent = buffer.nioBuffer(buffer.readerIndex(), length);
	// assume this event's content is read from the ByteBuf (positions are not shared!)
	buffer.readerIndex(buffer.readerIndex() + length);

	TaskEvent event =
		(TaskEvent) EventSerializer.fromSerializedEvent(serializedEvent, classLoader);

	ResultPartitionID partitionId =
		new ResultPartitionID(
			IntermediateResultPartitionID.fromByteBuf(buffer),
			ExecutionAttemptID.fromByteBuf(buffer));

	InputChannelID receiverId = InputChannelID.fromByteBuf(buffer);

	return new TaskEventRequest(event, partitionId, receiverId);
}
Example 4
Source File: BufferSpiller.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@Override
public BufferOrEvent getNext() throws IOException {
	if (buffer.remaining() < HEADER_LENGTH) {
		buffer.compact();

		while (buffer.position() < HEADER_LENGTH) {
			if (fileChannel.read(buffer) == -1) {
				if (buffer.position() == 0) {
					// no trailing data
					return null;
				} else {
					throw new IOException("Found trailing incomplete buffer or event");
				}
			}
		}

		buffer.flip();
	}

	final int channel = buffer.getInt();
	final int length = buffer.getInt();
	final boolean isBuffer = buffer.get() == 0;

	if (isBuffer) {
		// deserialize buffer
		if (length > pageSize) {
			throw new IOException(String.format(
					"Spilled buffer (%d bytes) is larger than page size of (%d bytes)", length, pageSize));
		}

		MemorySegment seg = MemorySegmentFactory.allocateUnpooledSegment(pageSize);

		int segPos = 0;
		int bytesRemaining = length;

		while (true) {
			int toCopy = Math.min(buffer.remaining(), bytesRemaining);
			if (toCopy > 0) {
				seg.put(segPos, buffer, toCopy);
				segPos += toCopy;
				bytesRemaining -= toCopy;
			}

			if (bytesRemaining == 0) {
				break;
			}
			else {
				buffer.clear();
				if (fileChannel.read(buffer) == -1) {
					throw new IOException("Found trailing incomplete buffer");
				}
				buffer.flip();
			}
		}

		Buffer buf = new NetworkBuffer(seg, FreeingBufferRecycler.INSTANCE);
		buf.setSize(length);

		return new BufferOrEvent(buf, channel);
	}
	else {
		// deserialize event
		if (length > buffer.capacity() - HEADER_LENGTH) {
			throw new IOException("Event is too large");
		}

		if (buffer.remaining() < length) {
			buffer.compact();

			while (buffer.position() < length) {
				if (fileChannel.read(buffer) == -1) {
					throw new IOException("Found trailing incomplete event");
				}
			}

			buffer.flip();
		}

		int oldLimit = buffer.limit();
		buffer.limit(buffer.position() + length);
		AbstractEvent evt = EventSerializer.fromSerializedEvent(buffer, getClass().getClassLoader());
		buffer.limit(oldLimit);

		return new BufferOrEvent(evt, channel);
	}
}
Example 5
Source File: BufferSpiller.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Override
public BufferOrEvent getNext() throws IOException {
	if (buffer.remaining() < HEADER_LENGTH) {
		buffer.compact();

		while (buffer.position() < HEADER_LENGTH) {
			if (fileChannel.read(buffer) == -1) {
				if (buffer.position() == 0) {
					// no trailing data
					return null;
				} else {
					throw new IOException("Found trailing incomplete buffer or event");
				}
			}
		}

		buffer.flip();
	}

	final int channel = buffer.getInt();
	final int length = buffer.getInt();
	final boolean isBuffer = buffer.get() == 0;

	if (isBuffer) {
		// deserialize buffer
		if (length > pageSize) {
			throw new IOException(String.format(
					"Spilled buffer (%d bytes) is larger than page size of (%d bytes)", length, pageSize));
		}

		MemorySegment seg = MemorySegmentFactory.allocateUnpooledSegment(pageSize);

		int segPos = 0;
		int bytesRemaining = length;

		while (true) {
			int toCopy = Math.min(buffer.remaining(), bytesRemaining);
			if (toCopy > 0) {
				seg.put(segPos, buffer, toCopy);
				segPos += toCopy;
				bytesRemaining -= toCopy;
			}

			if (bytesRemaining == 0) {
				break;
			}
			else {
				buffer.clear();
				if (fileChannel.read(buffer) == -1) {
					throw new IOException("Found trailing incomplete buffer");
				}
				buffer.flip();
			}
		}

		Buffer buf = new NetworkBuffer(seg, FreeingBufferRecycler.INSTANCE);
		buf.setSize(length);

		return new BufferOrEvent(buf, channel, true);
	}
	else {
		// deserialize event
		if (length > buffer.capacity() - HEADER_LENGTH) {
			throw new IOException("Event is too large");
		}

		if (buffer.remaining() < length) {
			buffer.compact();

			while (buffer.position() < length) {
				if (fileChannel.read(buffer) == -1) {
					throw new IOException("Found trailing incomplete event");
				}
			}

			buffer.flip();
		}

		int oldLimit = buffer.limit();
		buffer.limit(buffer.position() + length);
		AbstractEvent evt = EventSerializer.fromSerializedEvent(buffer, getClass().getClassLoader());
		buffer.limit(oldLimit);

		return new BufferOrEvent(evt, channel, true, length);
	}
}