org.apache.flink.runtime.io.network.api.serialization.EventSerializer Java Examples

static TaskEventRequest readFrom(ByteBuf buffer, ClassLoader classLoader) throws IOException {
	// directly deserialize fromNetty's buffer
	int length = buffer.readInt();
	ByteBuffer serializedEvent = buffer.nioBuffer(buffer.readerIndex(), length);
	// assume this event's content is read from the ByteBuf (positions are not shared!)
	buffer.readerIndex(buffer.readerIndex() + length);

	TaskEvent event =
		(TaskEvent) EventSerializer.fromSerializedEvent(serializedEvent, classLoader);

	ResultPartitionID partitionId =
		new ResultPartitionID(
			IntermediateResultPartitionID.fromByteBuf(buffer),
			ExecutionAttemptID.fromByteBuf(buffer));

	InputChannelID receiverId = InputChannelID.fromByteBuf(buffer);

	return new TaskEventRequest(event, partitionId, receiverId);
}
 

@Override
protected void deserializeBuffer(Buffer buffer) throws IOException {
	if (buffer.isBuffer()) {
		deserializer.setNextBuffer(buffer);

		while (deserializer.hasUnfinishedData()) {
			RecordDeserializer.DeserializationResult result =
				deserializer.getNextRecord(delegate);

			if (result.isFullRecord()) {
				output.add(delegate.getInstance());
			}

			if (result == RecordDeserializer.DeserializationResult.LAST_RECORD_FROM_BUFFER
				|| result == RecordDeserializer.DeserializationResult.PARTIAL_RECORD) {
				break;
			}
		}
	} else {
		// is event
		AbstractEvent event = EventSerializer.fromBuffer(buffer, getClass().getClassLoader());
		output.add(event);
	}
}
 

@Override
public synchronized void finish() throws IOException {
	synchronized (buffers) {
		if (add(EventSerializer.toBufferConsumer(EndOfPartitionEvent.INSTANCE), true)) {
			isFinished = true;
		}

		flush();
	}

	// If we are spilling/have spilled, wait for the writer to finish
	if (spillWriter != null) {
		spillWriter.close();
	}
	LOG.debug("{}: Finished {}.", parent.getOwningTaskName(), this);
}
 

@Override
protected void deserializeBuffer(Buffer buffer) throws IOException {
	if (buffer.isBuffer()) {
		deserializer.setNextBuffer(buffer);

		while (deserializer.hasUnfinishedData()) {
			RecordDeserializer.DeserializationResult result =
				deserializer.getNextRecord(delegate);

			if (result.isFullRecord()) {
				output.add(delegate.getInstance());
			}

			if (result == RecordDeserializer.DeserializationResult.LAST_RECORD_FROM_BUFFER
				|| result == RecordDeserializer.DeserializationResult.PARTIAL_RECORD) {
				break;
			}
		}
	} else {
		// is event
		AbstractEvent event = EventSerializer.fromBuffer(buffer, getClass().getClassLoader());
		output.add(event);
	}
}
 

@Override
protected void deserializeBuffer(Buffer buffer) throws IOException {
	if (buffer.isBuffer()) {
		deserializer.setNextBuffer(buffer);

		while (deserializer.hasUnfinishedData()) {
			RecordDeserializer.DeserializationResult result =
				deserializer.getNextRecord(delegate);

			if (result.isFullRecord()) {
				output.add(delegate.getInstance());
			}

			if (result == RecordDeserializer.DeserializationResult.LAST_RECORD_FROM_BUFFER
				|| result == RecordDeserializer.DeserializationResult.PARTIAL_RECORD) {
				break;
			}
		}
	} else {
		// is event
		AbstractEvent event = EventSerializer.fromBuffer(buffer, getClass().getClassLoader());
		output.add(event);
	}
}
 

private static BufferOrEvent generateAndWriteEvent(FileChannel fileChannel, Random rnd, int numberOfChannels) throws IOException {
	long magicNumber = rnd.nextLong();
	byte[] data = new byte[rnd.nextInt(1000)];
	rnd.nextBytes(data);
	TestEvent evt = new TestEvent(magicNumber, data);

	int channelIndex = rnd.nextInt(numberOfChannels);

	ByteBuffer serializedEvent = EventSerializer.toSerializedEvent(evt);
	ByteBuffer header = ByteBuffer.allocate(9);
	header.order(ByteOrder.LITTLE_ENDIAN);

	header.putInt(channelIndex);
	header.putInt(serializedEvent.remaining());
	header.put((byte) 1);
	header.flip();

	FileUtils.writeCompletely(fileChannel, header);
	FileUtils.writeCompletely(fileChannel, serializedEvent);
	return new BufferOrEvent(evt, channelIndex);
}
 

static TaskEventRequest readFrom(ByteBuf buffer, ClassLoader classLoader) throws IOException {
	// directly deserialize fromNetty's buffer
	int length = buffer.readInt();
	ByteBuffer serializedEvent = buffer.nioBuffer(buffer.readerIndex(), length);
	// assume this event's content is read from the ByteBuf (positions are not shared!)
	buffer.readerIndex(buffer.readerIndex() + length);

	TaskEvent event =
		(TaskEvent) EventSerializer.fromSerializedEvent(serializedEvent, classLoader);

	ResultPartitionID partitionId =
		new ResultPartitionID(
			IntermediateResultPartitionID.fromByteBuf(buffer),
			ExecutionAttemptID.fromByteBuf(buffer));

	InputChannelID receiverId = InputChannelID.fromByteBuf(buffer);

	return new TaskEventRequest(event, partitionId, receiverId);
}
 

private static BufferOrEvent generateAndWriteEvent(FileChannel fileChannel, Random rnd, int numberOfChannels) throws IOException {
	long magicNumber = rnd.nextLong();
	byte[] data = new byte[rnd.nextInt(1000)];
	rnd.nextBytes(data);
	TestEvent evt = new TestEvent(magicNumber, data);

	int channelIndex = rnd.nextInt(numberOfChannels);

	ByteBuffer serializedEvent = EventSerializer.toSerializedEvent(evt);
	ByteBuffer header = ByteBuffer.allocate(9);
	header.order(ByteOrder.LITTLE_ENDIAN);

	header.putInt(channelIndex);
	header.putInt(serializedEvent.remaining());
	header.put((byte) 1);
	header.flip();

	FileUtils.writeCompletely(fileChannel, header);
	FileUtils.writeCompletely(fileChannel, serializedEvent);
	return new BufferOrEvent(evt, channelIndex);
}
 

static TaskEventRequest readFrom(ByteBuf buffer, ClassLoader classLoader) throws IOException {
	// directly deserialize fromNetty's buffer
	int length = buffer.readInt();
	ByteBuffer serializedEvent = buffer.nioBuffer(buffer.readerIndex(), length);
	// assume this event's content is read from the ByteBuf (positions are not shared!)
	buffer.readerIndex(buffer.readerIndex() + length);

	TaskEvent event =
		(TaskEvent) EventSerializer.fromSerializedEvent(serializedEvent, classLoader);

	ResultPartitionID partitionId =
		new ResultPartitionID(
			IntermediateResultPartitionID.fromByteBuf(buffer),
			ExecutionAttemptID.fromByteBuf(buffer));

	InputChannelID receiverId = InputChannelID.fromByteBuf(buffer);

	return new TaskEventRequest(event, partitionId, receiverId);
}
 

/**
 * Parses the buffer as an event and returns the {@link CheckpointBarrier} if the event is indeed a barrier or
 * returns null in all other cases.
 */
@Nullable
protected CheckpointBarrier parseCheckpointBarrierOrNull(Buffer buffer) throws IOException {
	if (buffer.isBuffer()) {
		return null;
	}

	AbstractEvent event = EventSerializer.fromBuffer(buffer, getClass().getClassLoader());
	// reset the buffer because it would be deserialized again in SingleInputGate while getting next buffer.
	// we can further improve to avoid double deserialization in the future.
	buffer.setReaderIndex(0);
	return event.getClass() == CheckpointBarrier.class ? (CheckpointBarrier) event : null;
}
 

TestInputChannel readEndOfPartitionEvent() {
	addBufferAndAvailability(
		() -> {
			setReleased();
			return Optional.of(new BufferAndAvailability(EventSerializer.toBuffer(EndOfPartitionEvent.INSTANCE),
				false,
				0));
		}
	);
	return this;
}
 

private IteratorWrappingTestSingleInputGate<T> wrapIterator(MutableObjectIterator<T> iterator) throws IOException, InterruptedException {
	inputIterator = iterator;
	serializer = new SpanningRecordSerializer<T>();

	// The input iterator can produce an infinite stream. That's why we have to serialize each
	// record on demand and cannot do it upfront.
	final BufferAndAvailabilityProvider answer = new BufferAndAvailabilityProvider() {

		private boolean hasData = inputIterator.next(reuse) != null;

		@Override
		public Optional<BufferAndAvailability> getBufferAvailability() throws IOException {
			if (hasData) {
				serializer.serializeRecord(reuse);
				BufferBuilder bufferBuilder = createBufferBuilder(bufferSize);
				serializer.copyToBufferBuilder(bufferBuilder);

				hasData = inputIterator.next(reuse) != null;

				// Call getCurrentBuffer to ensure size is set
				return Optional.of(new BufferAndAvailability(buildSingleBuffer(bufferBuilder), true, 0));
			} else {
				inputChannel.setReleased();

				return Optional.of(new BufferAndAvailability(EventSerializer.toBuffer(EndOfPartitionEvent.INSTANCE),
					false,
					0));
			}
		}
	};

	inputChannel.addBufferAndAvailability(answer);

	inputGate.setInputChannel(new IntermediateResultPartitionID(), inputChannel);

	return this;
}
 

@Test
public void testForwardsIsBuffer() throws IOException {
	assertEquals(buffer.isBuffer(), buffer.readOnlySlice().isBuffer());
	assertEquals(buffer.isBuffer(), buffer.readOnlySlice(1, 2).isBuffer());
	Buffer eventBuffer = EventSerializer.toBuffer(EndOfPartitionEvent.INSTANCE);
	assertEquals(eventBuffer.isBuffer(), eventBuffer.readOnlySlice().isBuffer());
	assertEquals(eventBuffer.isBuffer(), eventBuffer.readOnlySlice(1, 2).isBuffer());
}
 

@Override
public void add(BufferOrEvent boe) throws IOException {
	try {
		ByteBuffer contents;
		if (boe.isBuffer()) {
			Buffer buf = boe.getBuffer();
			contents = buf.getNioBufferReadable();
		}
		else {
			contents = EventSerializer.toSerializedEvent(boe.getEvent());
		}

		headBuffer.clear();
		headBuffer.putInt(boe.getChannelIndex());
		headBuffer.putInt(contents.remaining());
		headBuffer.put((byte) (boe.isBuffer() ? 0 : 1));
		headBuffer.flip();

		bytesWritten += (headBuffer.remaining() + contents.remaining());

		FileUtils.writeCompletely(currentChannel, headBuffer);
		FileUtils.writeCompletely(currentChannel, contents);
	}
	finally {
		if (boe.isBuffer()) {
			boe.getBuffer().recycleBuffer();
		}
	}
}
 

private static String buildBufferTypeErrorMessage(Buffer buffer) {
	try {
		AbstractEvent event = EventSerializer.fromBuffer(buffer, ChannelStateWriterImpl.class.getClassLoader());
		return String.format("Should be buffer but [%s] found", event);
	}
	catch (Exception ex) {
		return "Should be buffer";
	}
}
 

public void broadcastEvent(AbstractEvent event, boolean isPriorityEvent) throws IOException {
	try (BufferConsumer eventBufferConsumer = EventSerializer.toBufferConsumer(event)) {
		for (int targetChannel = 0; targetChannel < numberOfChannels; targetChannel++) {
			tryFinishCurrentBufferBuilder(targetChannel);

			// Retain the buffer so that it can be recycled by each channel of targetPartition
			targetPartition.addBufferConsumer(eventBufferConsumer.copy(), targetChannel, isPriorityEvent);
		}

		if (flushAlways) {
			flushAll();
		}
	}
}
 

@Override
ByteBuf write(ByteBufAllocator allocator) throws IOException {
	ByteBuf result = null;

	try {
		// TODO Directly serialize to Netty's buffer
		ByteBuffer serializedEvent = EventSerializer.toSerializedEvent(event);

		result = allocateBuffer(allocator, ID, 4 + serializedEvent.remaining() + 20 + 16 + 16);

		result.writeInt(serializedEvent.remaining());
		result.writeBytes(serializedEvent);

		partitionId.getPartitionId().writeTo(result);
		partitionId.getProducerId().writeTo(result);

		receiverId.writeTo(result);

		return result;
	}
	catch (Throwable t) {
		if (result != null) {
			result.release();
		}

		throw new IOException(t);
	}
}
 

private boolean isEndOfChannelStateEvent(Buffer buffer) throws IOException {
	if (buffer.isBuffer()) {
		return false;
	}

	AbstractEvent event = EventSerializer.fromBuffer(buffer, getClass().getClassLoader());
	buffer.setReaderIndex(0);
	return event.getClass() == EndOfChannelStateEvent.class;
}
 

@Override
ByteBuf write(ByteBufAllocator allocator) throws IOException {
	ByteBuf result = null;

	try {
		// TODO Directly serialize to Netty's buffer
		ByteBuffer serializedEvent = EventSerializer.toSerializedEvent(event);

		result = allocateBuffer(allocator, ID, 4 + serializedEvent.remaining() + 16 + 16 + 16);

		result.writeInt(serializedEvent.remaining());
		result.writeBytes(serializedEvent);

		partitionId.getPartitionId().writeTo(result);
		partitionId.getProducerId().writeTo(result);

		receiverId.writeTo(result);

		return result;
	}
	catch (Throwable t) {
		if (result != null) {
			result.release();
		}

		throw new IOException(t);
	}
}
 

private BufferOrEvent transformEvent(
		Buffer buffer,
		boolean moreAvailable,
		InputChannel currentChannel) throws IOException, InterruptedException {
	final AbstractEvent event;
	try {
		event = EventSerializer.fromBuffer(buffer, getClass().getClassLoader());
	} finally {
		buffer.recycleBuffer();
	}

	if (event.getClass() == EndOfPartitionEvent.class) {
		channelsWithEndOfPartitionEvents.set(currentChannel.getChannelIndex());

		if (channelsWithEndOfPartitionEvents.cardinality() == numberOfInputChannels) {
			// Because of race condition between:
			// 1. releasing inputChannelsWithData lock in this method and reaching this place
			// 2. empty data notification that re-enqueues a channel
			// we can end up with moreAvailable flag set to true, while we expect no more data.
			checkState(!moreAvailable || !pollNext().isPresent());
			moreAvailable = false;
			hasReceivedAllEndOfPartitionEvents = true;
			markAvailable();
		}

		currentChannel.releaseAllResources();
	}

	return new BufferOrEvent(event, currentChannel.getChannelInfo(), moreAvailable, buffer.getSize());
}
 

@Override
public void finish() throws IOException {
	checkState(!isReleased, "data partition already released");
	checkState(!isFinished, "data partition already finished");

	isFinished = true;
	flushCurrentBuffer();
	writeAndCloseBufferConsumer(EventSerializer.toBufferConsumer(EndOfPartitionEvent.INSTANCE));
	data.finishWrite();
}
 

static BufferOrEvent parseBuffer(BufferConsumer bufferConsumer, int targetChannel) throws IOException {
	Buffer buffer = buildSingleBuffer(bufferConsumer);
	if (buffer.isBuffer()) {
		return new BufferOrEvent(buffer, new InputChannelInfo(0, targetChannel));
	} else {
		// is event:
		AbstractEvent event = EventSerializer.fromBuffer(buffer, RecordWriterTest.class.getClassLoader());
		buffer.recycleBuffer(); // the buffer is not needed anymore
		return new BufferOrEvent(event, new InputChannelInfo(0, targetChannel));
	}
}
 

@Test
public void testBarrierOvertaking() throws Exception {
	subpartition.add(createFilledFinishedBufferConsumer(1));
	assertEquals(0, availablityListener.getNumNotifications());
	assertEquals(0, availablityListener.getNumPriorityEvents());

	subpartition.add(createFilledFinishedBufferConsumer(2));
	assertEquals(1, availablityListener.getNumNotifications());
	assertEquals(0, availablityListener.getNumPriorityEvents());

	BufferConsumer eventBuffer = EventSerializer.toBufferConsumer(EndOfSuperstepEvent.INSTANCE);
	subpartition.add(eventBuffer);
	assertEquals(1, availablityListener.getNumNotifications());
	assertEquals(0, availablityListener.getNumPriorityEvents());

	subpartition.add(createFilledFinishedBufferConsumer(4));
	assertEquals(1, availablityListener.getNumNotifications());
	assertEquals(0, availablityListener.getNumPriorityEvents());

	CheckpointOptions options = new CheckpointOptions(
		CheckpointType.CHECKPOINT,
		new CheckpointStorageLocationReference(new byte[]{0, 1, 2}),
		true,
		true);
	BufferConsumer barrierBuffer = EventSerializer.toBufferConsumer(new CheckpointBarrier(0, 0, options));
	subpartition.add(barrierBuffer, true);
	assertEquals(2, availablityListener.getNumNotifications());
	assertEquals(0, availablityListener.getNumPriorityEvents());

	List<Buffer> inflight = subpartition.requestInflightBufferSnapshot();
	assertEquals(Arrays.asList(1, 2, 4), inflight.stream().map(Buffer::getSize).collect(Collectors.toList()));
	inflight.forEach(Buffer::recycleBuffer);

	assertNextEvent(readView, barrierBuffer.getWrittenBytes(), CheckpointBarrier.class, true, 2, false, true);
	assertNextBuffer(readView, 1, true, 1, false, true);
	assertNextBuffer(readView, 2, true, 0, true, true);
	assertNextEvent(readView, eventBuffer.getWrittenBytes(), EndOfSuperstepEvent.class, false, 0, false, true);
	assertNextBuffer(readView, 4, false, 0, false, true);
	assertNoNextBuffer(readView);
}
 

private static void assertNextBufferOrEvent(
		ResultSubpartitionView readView,
		int expectedReadableBufferSize,
		boolean expectedIsBuffer,
		@Nullable Class<? extends AbstractEvent> expectedEventClass,
		boolean expectedIsDataAvailable,
		int expectedBuffersInBacklog,
		boolean expectedIsEventAvailable,
		boolean expectedRecycledAfterRecycle) throws IOException, InterruptedException {
	checkArgument(expectedEventClass == null || !expectedIsBuffer);

	ResultSubpartition.BufferAndBacklog bufferAndBacklog = readView.getNextBuffer();
	assertNotNull(bufferAndBacklog);
	try {
		assertEquals("buffer size", expectedReadableBufferSize,
				bufferAndBacklog.buffer().readableBytes());
		assertEquals("buffer or event", expectedIsBuffer,
				bufferAndBacklog.buffer().isBuffer());
		if (expectedEventClass != null) {
			Assert.assertThat(EventSerializer
							.fromBuffer(bufferAndBacklog.buffer(), ClassLoader.getSystemClassLoader()),
					instanceOf(expectedEventClass));
		}
		assertEquals("data available", expectedIsDataAvailable,
				bufferAndBacklog.isDataAvailable());
		assertEquals("data available", expectedIsDataAvailable, readView.isAvailable(Integer.MAX_VALUE));
		assertEquals("backlog", expectedBuffersInBacklog, bufferAndBacklog.buffersInBacklog());
		assertEquals("event available", expectedIsEventAvailable, bufferAndBacklog.isEventAvailable());
		assertEquals("event available", expectedIsEventAvailable, readView.isAvailable(0));

		assertFalse("not recycled", bufferAndBacklog.buffer().isRecycled());
	} finally {
		bufferAndBacklog.buffer().recycleBuffer();
	}
	assertEquals("recycled", expectedRecycledAfterRecycle, bufferAndBacklog.buffer().isRecycled());
}
 

TestInputChannel readEndOfPartitionEvent() {
	addBufferAndAvailability(
		() -> {
			setReleased();
			return Optional.of(new BufferAndAvailability(EventSerializer.toBuffer(EndOfPartitionEvent.INSTANCE),
				false,
				0));
		}
	);
	return this;
}
 

private IteratorWrappingTestSingleInputGate<T> wrapIterator(MutableObjectIterator<T> iterator) throws IOException, InterruptedException {
	inputIterator = iterator;
	serializer = new SpanningRecordSerializer<T>();

	// The input iterator can produce an infinite stream. That's why we have to serialize each
	// record on demand and cannot do it upfront.
	final BufferAndAvailabilityProvider answer = new BufferAndAvailabilityProvider() {

		private boolean hasData = inputIterator.next(reuse) != null;

		@Override
		public Optional<BufferAndAvailability> getBufferAvailability() throws IOException {
			if (hasData) {
				serializer.serializeRecord(reuse);
				BufferBuilder bufferBuilder = createBufferBuilder(bufferSize);
				BufferConsumer bufferConsumer = bufferBuilder.createBufferConsumer();
				serializer.copyToBufferBuilder(bufferBuilder);

				hasData = inputIterator.next(reuse) != null;

				// Call getCurrentBuffer to ensure size is set
				return Optional.of(new BufferAndAvailability(bufferConsumer.build(), true, 0));
			} else {
				inputChannel.setReleased();

				return Optional.of(new BufferAndAvailability(EventSerializer.toBuffer(EndOfPartitionEvent.INSTANCE),
					false,
					0));
			}
		}
	};

	inputChannel.addBufferAndAvailability(answer);

	inputGate.setInputChannels(inputChannel);

	return this;
}
 

@Test
public void testForwardsIsBuffer() throws IOException {
	assertEquals(buffer.isBuffer(), buffer.readOnlySlice().isBuffer());
	assertEquals(buffer.isBuffer(), buffer.readOnlySlice(1, 2).isBuffer());
	Buffer eventBuffer = EventSerializer.toBuffer(EndOfPartitionEvent.INSTANCE);
	assertEquals(eventBuffer.isBuffer(), eventBuffer.readOnlySlice().isBuffer());
	assertEquals(eventBuffer.isBuffer(), eventBuffer.readOnlySlice(1, 2).isBuffer());
}
 

private static BufferOrEvent parseBuffer(BufferConsumer bufferConsumer, int targetChannel) throws IOException {
	Buffer buffer = buildSingleBuffer(bufferConsumer);
	if (buffer.isBuffer()) {
		return new BufferOrEvent(buffer, targetChannel);
	} else {
		// is event:
		AbstractEvent event = EventSerializer.fromBuffer(buffer, RecordWriterTest.class.getClassLoader());
		buffer.recycleBuffer(); // the buffer is not needed anymore
		return new BufferOrEvent(event, targetChannel);
	}
}
 

public void broadcastEvent(AbstractEvent event) throws IOException {
	try (BufferConsumer eventBufferConsumer = EventSerializer.toBufferConsumer(event)) {
		for (int targetChannel = 0; targetChannel < numberOfChannels; targetChannel++) {
			tryFinishCurrentBufferBuilder(targetChannel);

			// Retain the buffer so that it can be recycled by each channel of targetPartition
			targetPartition.addBufferConsumer(eventBufferConsumer.copy(), targetChannel);
		}

		if (flushAlways) {
			flushAll();
		}
	}
}
 

private static BufferOrEvent parseBuffer(BufferConsumer bufferConsumer, int targetChannel) throws IOException {
	Buffer buffer = buildSingleBuffer(bufferConsumer);
	if (buffer.isBuffer()) {
		return new BufferOrEvent(buffer, targetChannel);
	} else {
		// is event:
		AbstractEvent event = EventSerializer.fromBuffer(buffer, RecordWriterTest.class.getClassLoader());
		buffer.recycleBuffer(); // the buffer is not needed anymore
		return new BufferOrEvent(event, targetChannel);
	}
}