Java Code Examples for org.apache.flink.runtime.io.network.api.serialization.EventSerializer#fromBuffer()

@Override
protected void deserializeBuffer(Buffer buffer) throws IOException {
	if (buffer.isBuffer()) {
		deserializer.setNextBuffer(buffer);

		while (deserializer.hasUnfinishedData()) {
			RecordDeserializer.DeserializationResult result =
				deserializer.getNextRecord(delegate);

			if (result.isFullRecord()) {
				output.add(delegate.getInstance());
			}

			if (result == RecordDeserializer.DeserializationResult.LAST_RECORD_FROM_BUFFER
				|| result == RecordDeserializer.DeserializationResult.PARTIAL_RECORD) {
				break;
			}
		}
	} else {
		// is event
		AbstractEvent event = EventSerializer.fromBuffer(buffer, getClass().getClassLoader());
		output.add(event);
	}
}
 

@Override
protected void deserializeBuffer(Buffer buffer) throws IOException {
	if (buffer.isBuffer()) {
		deserializer.setNextBuffer(buffer);

		while (deserializer.hasUnfinishedData()) {
			RecordDeserializer.DeserializationResult result =
				deserializer.getNextRecord(delegate);

			if (result.isFullRecord()) {
				output.add(delegate.getInstance());
			}

			if (result == RecordDeserializer.DeserializationResult.LAST_RECORD_FROM_BUFFER
				|| result == RecordDeserializer.DeserializationResult.PARTIAL_RECORD) {
				break;
			}
		}
	} else {
		// is event
		AbstractEvent event = EventSerializer.fromBuffer(buffer, getClass().getClassLoader());
		output.add(event);
	}
}
 

@Override
protected void deserializeBuffer(Buffer buffer) throws IOException {
	if (buffer.isBuffer()) {
		deserializer.setNextBuffer(buffer);

		while (deserializer.hasUnfinishedData()) {
			RecordDeserializer.DeserializationResult result =
				deserializer.getNextRecord(delegate);

			if (result.isFullRecord()) {
				output.add(delegate.getInstance());
			}

			if (result == RecordDeserializer.DeserializationResult.LAST_RECORD_FROM_BUFFER
				|| result == RecordDeserializer.DeserializationResult.PARTIAL_RECORD) {
				break;
			}
		}
	} else {
		// is event
		AbstractEvent event = EventSerializer.fromBuffer(buffer, getClass().getClassLoader());
		output.add(event);
	}
}
 

private static BufferOrEvent parseBuffer(BufferConsumer bufferConsumer, int targetChannel) throws IOException {
	Buffer buffer = buildSingleBuffer(bufferConsumer);
	if (buffer.isBuffer()) {
		return new BufferOrEvent(buffer, targetChannel);
	} else {
		// is event:
		AbstractEvent event = EventSerializer.fromBuffer(buffer, RecordWriterTest.class.getClassLoader());
		buffer.recycleBuffer(); // the buffer is not needed anymore
		return new BufferOrEvent(event, targetChannel);
	}
}
 

private BufferOrEvent transformToBufferOrEvent(
		Buffer buffer,
		boolean moreAvailable,
		InputChannel currentChannel) throws IOException, InterruptedException {
	if (buffer.isBuffer()) {
		return new BufferOrEvent(buffer, currentChannel.getChannelIndex(), moreAvailable);
	}
	else {
		final AbstractEvent event;
		try {
			event = EventSerializer.fromBuffer(buffer, getClass().getClassLoader());
		}
		finally {
			buffer.recycleBuffer();
		}

		if (event.getClass() == EndOfPartitionEvent.class) {
			channelsWithEndOfPartitionEvents.set(currentChannel.getChannelIndex());

			if (channelsWithEndOfPartitionEvents.cardinality() == numberOfInputChannels) {
				// Because of race condition between:
				// 1. releasing inputChannelsWithData lock in this method and reaching this place
				// 2. empty data notification that re-enqueues a channel
				// we can end up with moreAvailable flag set to true, while we expect no more data.
				checkState(!moreAvailable || !pollNext().isPresent());
				moreAvailable = false;
				hasReceivedAllEndOfPartitionEvents = true;
				markAvailable();
			}

			currentChannel.notifySubpartitionConsumed();
			currentChannel.releaseAllResources();
		}

		return new BufferOrEvent(event, currentChannel.getChannelIndex(), moreAvailable, buffer.getSize());
	}
}
 

private static BufferOrEvent parseBuffer(BufferConsumer bufferConsumer, int targetChannel) throws IOException {
	Buffer buffer = buildSingleBuffer(bufferConsumer);
	if (buffer.isBuffer()) {
		return new BufferOrEvent(buffer, targetChannel);
	} else {
		// is event:
		AbstractEvent event = EventSerializer.fromBuffer(buffer, RecordWriterTest.class.getClassLoader());
		buffer.recycleBuffer(); // the buffer is not needed anymore
		return new BufferOrEvent(event, targetChannel);
	}
}
 

private static String buildBufferTypeErrorMessage(Buffer buffer) {
	try {
		AbstractEvent event = EventSerializer.fromBuffer(buffer, ChannelStateWriterImpl.class.getClassLoader());
		return String.format("Should be buffer but [%s] found", event);
	}
	catch (Exception ex) {
		return "Should be buffer";
	}
}
 

private boolean isEndOfChannelStateEvent(Buffer buffer) throws IOException {
	if (buffer.isBuffer()) {
		return false;
	}

	AbstractEvent event = EventSerializer.fromBuffer(buffer, getClass().getClassLoader());
	buffer.setReaderIndex(0);
	return event.getClass() == EndOfChannelStateEvent.class;
}
 

/**
 * Parses the buffer as an event and returns the {@link CheckpointBarrier} if the event is indeed a barrier or
 * returns null in all other cases.
 */
@Nullable
protected CheckpointBarrier parseCheckpointBarrierOrNull(Buffer buffer) throws IOException {
	if (buffer.isBuffer()) {
		return null;
	}

	AbstractEvent event = EventSerializer.fromBuffer(buffer, getClass().getClassLoader());
	// reset the buffer because it would be deserialized again in SingleInputGate while getting next buffer.
	// we can further improve to avoid double deserialization in the future.
	buffer.setReaderIndex(0);
	return event.getClass() == CheckpointBarrier.class ? (CheckpointBarrier) event : null;
}
 

private BufferOrEvent transformEvent(
		Buffer buffer,
		boolean moreAvailable,
		InputChannel currentChannel) throws IOException, InterruptedException {
	final AbstractEvent event;
	try {
		event = EventSerializer.fromBuffer(buffer, getClass().getClassLoader());
	} finally {
		buffer.recycleBuffer();
	}

	if (event.getClass() == EndOfPartitionEvent.class) {
		channelsWithEndOfPartitionEvents.set(currentChannel.getChannelIndex());

		if (channelsWithEndOfPartitionEvents.cardinality() == numberOfInputChannels) {
			// Because of race condition between:
			// 1. releasing inputChannelsWithData lock in this method and reaching this place
			// 2. empty data notification that re-enqueues a channel
			// we can end up with moreAvailable flag set to true, while we expect no more data.
			checkState(!moreAvailable || !pollNext().isPresent());
			moreAvailable = false;
			hasReceivedAllEndOfPartitionEvents = true;
			markAvailable();
		}

		currentChannel.releaseAllResources();
	}

	return new BufferOrEvent(event, currentChannel.getChannelInfo(), moreAvailable, buffer.getSize());
}
 

static BufferOrEvent parseBuffer(BufferConsumer bufferConsumer, int targetChannel) throws IOException {
	Buffer buffer = buildSingleBuffer(bufferConsumer);
	if (buffer.isBuffer()) {
		return new BufferOrEvent(buffer, new InputChannelInfo(0, targetChannel));
	} else {
		// is event:
		AbstractEvent event = EventSerializer.fromBuffer(buffer, RecordWriterTest.class.getClassLoader());
		buffer.recycleBuffer(); // the buffer is not needed anymore
		return new BufferOrEvent(event, new InputChannelInfo(0, targetChannel));
	}
}
 

private Optional<BufferOrEvent> getNextBufferOrEvent(boolean blocking) throws IOException, InterruptedException {
	if (hasReceivedAllEndOfPartitionEvents) {
		return Optional.empty();
	}

	if (isReleased) {
		throw new IllegalStateException("Released");
	}

	requestPartitions();

	InputChannel currentChannel;
	boolean moreAvailable;
	Optional<BufferAndAvailability> result = Optional.empty();

	do {
		synchronized (inputChannelsWithData) {
			while (inputChannelsWithData.size() == 0) {
				if (isReleased) {
					throw new IllegalStateException("Released");
				}

				if (blocking) {
					inputChannelsWithData.wait();
				}
				else {
					return Optional.empty();
				}
			}

			currentChannel = inputChannelsWithData.remove();
			enqueuedInputChannelsWithData.clear(currentChannel.getChannelIndex());
			moreAvailable = !inputChannelsWithData.isEmpty();
		}

		result = currentChannel.getNextBuffer();
	} while (!result.isPresent());

	// this channel was now removed from the non-empty channels queue
	// we re-add it in case it has more data, because in that case no "non-empty" notification
	// will come for that channel
	if (result.get().moreAvailable()) {
		queueChannel(currentChannel);
		moreAvailable = true;
	}

	final Buffer buffer = result.get().buffer();
	if (buffer.isBuffer()) {
		return Optional.of(new BufferOrEvent(buffer, currentChannel.getChannelIndex(), moreAvailable));
	}
	else {
		final AbstractEvent event = EventSerializer.fromBuffer(buffer, getClass().getClassLoader());

		if (event.getClass() == EndOfPartitionEvent.class) {
			channelsWithEndOfPartitionEvents.set(currentChannel.getChannelIndex());

			if (channelsWithEndOfPartitionEvents.cardinality() == numberOfInputChannels) {
				// Because of race condition between:
				// 1. releasing inputChannelsWithData lock in this method and reaching this place
				// 2. empty data notification that re-enqueues a channel
				// we can end up with moreAvailable flag set to true, while we expect no more data.
				checkState(!moreAvailable || !pollNextBufferOrEvent().isPresent());
				moreAvailable = false;
				hasReceivedAllEndOfPartitionEvents = true;
			}

			currentChannel.notifySubpartitionConsumed();

			currentChannel.releaseAllResources();
		}

		return Optional.of(new BufferOrEvent(event, currentChannel.getChannelIndex(), moreAvailable));
	}
}
 

@Override
public Boolean call() throws Exception {
	try {
		while (true) {
			if (Thread.interrupted()) {
				throw new InterruptedException();
			}

			synchronized (dataAvailableNotification) {
				while (!dataAvailableNotification.getAndSet(false)) {
					dataAvailableNotification.wait();
				}
			}

			final BufferAndBacklog bufferAndBacklog = subpartitionView.getNextBuffer();

			if (isSlowConsumer) {
				Thread.sleep(random.nextInt(MAX_SLEEP_TIME_MS + 1));
			}

			if (bufferAndBacklog != null) {
				if (bufferAndBacklog.isMoreAvailable()) {
					dataAvailableNotification.set(true);
				}
				if (bufferAndBacklog.buffer().isBuffer()) {
					callback.onBuffer(bufferAndBacklog.buffer());
				} else {
					final AbstractEvent event = EventSerializer.fromBuffer(bufferAndBacklog.buffer(),
						getClass().getClassLoader());

					callback.onEvent(event);

					bufferAndBacklog.buffer().recycleBuffer();

					if (event.getClass() == EndOfPartitionEvent.class) {
						subpartitionView.notifySubpartitionConsumed();

						return true;
					}
				}
			} else if (subpartitionView.isReleased()) {
				return true;
			}
		}
	} finally {
		subpartitionView.releaseAllResources();
	}
}
 

@Override
public Boolean call() throws Exception {
	try {
		while (true) {
			if (Thread.interrupted()) {
				throw new InterruptedException();
			}

			synchronized (dataAvailableNotification) {
				while (!dataAvailableNotification.getAndSet(false)) {
					dataAvailableNotification.wait();
				}
			}

			final BufferAndBacklog bufferAndBacklog = subpartitionView.getNextBuffer();

			if (isSlowConsumer) {
				Thread.sleep(random.nextInt(MAX_SLEEP_TIME_MS + 1));
			}

			if (bufferAndBacklog != null) {
				if (bufferAndBacklog.isMoreAvailable()) {
					dataAvailableNotification.set(true);
				}
				if (bufferAndBacklog.buffer().isBuffer()) {
					callback.onBuffer(bufferAndBacklog.buffer());
				} else {
					final AbstractEvent event = EventSerializer.fromBuffer(bufferAndBacklog.buffer(),
						getClass().getClassLoader());

					callback.onEvent(event);

					bufferAndBacklog.buffer().recycleBuffer();

					if (event.getClass() == EndOfPartitionEvent.class) {
						subpartitionView.notifySubpartitionConsumed();

						return true;
					}
				}
			} else if (subpartitionView.isReleased()) {
				return true;
			}
		}
	} finally {
		subpartitionView.releaseAllResources();
	}
}
 

@Override
public Boolean call() throws Exception {
	try {
		while (true) {
			if (Thread.interrupted()) {
				throw new InterruptedException();
			}

			synchronized (dataAvailableNotification) {
				while (!dataAvailableNotification.getAndSet(false)) {
					dataAvailableNotification.wait();
				}
			}

			final BufferAndBacklog bufferAndBacklog = subpartitionView.getNextBuffer();

			if (isSlowConsumer) {
				Thread.sleep(random.nextInt(MAX_SLEEP_TIME_MS + 1));
			}

			if (bufferAndBacklog != null) {
				if (bufferAndBacklog.isDataAvailable()) {
					dataAvailableNotification.set(true);
				}
				if (bufferAndBacklog.buffer().isBuffer()) {
					callback.onBuffer(bufferAndBacklog.buffer());
				} else {
					final AbstractEvent event = EventSerializer.fromBuffer(bufferAndBacklog.buffer(),
						getClass().getClassLoader());

					callback.onEvent(event);

					bufferAndBacklog.buffer().recycleBuffer();

					if (event.getClass() == EndOfPartitionEvent.class) {
						subpartitionView.releaseAllResources();

						return true;
					}
				}
			} else if (subpartitionView.isReleased()) {
				return true;
			}
		}
	} finally {
		subpartitionView.releaseAllResources();
	}
}