org.apache.flink.runtime.io.network.api.serialization.RecordDeserializer.DeserializationResult Java Examples

@Override
@Nullable
public StreamElement pollNextNullable() throws Exception {

	while (true) {
		// get the stream element from the deserializer
		if (currentRecordDeserializer != null) {
			DeserializationResult result = currentRecordDeserializer.getNextRecord(deserializationDelegate);
			if (result.isBufferConsumed()) {
				currentRecordDeserializer.getCurrentBuffer().recycleBuffer();
				currentRecordDeserializer = null;
			}

			if (result.isFullRecord()) {
				return deserializationDelegate.getInstance();
			}
		}

		Optional<BufferOrEvent> bufferOrEvent = checkpointedInputGate.pollNext();
		if (bufferOrEvent.isPresent()) {
			processBufferOrEvent(bufferOrEvent.get());
		} else {
			if (checkpointedInputGate.isFinished()) {
				isFinished = true;
				checkState(checkpointedInputGate.isAvailable().isDone(), "Finished BarrierHandler should be available");
				if (!checkpointedInputGate.isEmpty()) {
					throw new IllegalStateException("Trailing data in checkpoint barrier handler.");
				}
			}
			return null;
		}
	}
}
 

DeserializationResult readInto(IOReadableWritable target) throws IOException {
	try {
		target.read(this);
	} catch (IndexOutOfBoundsException e) {
		throw new IOException(BROKEN_SERIALIZATION_ERROR_MESSAGE, e);
	}
	int remaining = remaining();
	if (remaining < 0) {
		throw new IOException(BROKEN_SERIALIZATION_ERROR_MESSAGE, new IndexOutOfBoundsException("Remaining = " + remaining));
	}
	return remaining == 0 ? LAST_RECORD_FROM_BUFFER : INTERMEDIATE_RECORD_FROM_BUFFER;
}
 

@Override
public InputStatus emitNext(DataOutput<T> output) throws Exception {

	while (true) {
		// get the stream element from the deserializer
		if (currentRecordDeserializer != null) {
			DeserializationResult result = currentRecordDeserializer.getNextRecord(deserializationDelegate);
			if (result.isBufferConsumed()) {
				currentRecordDeserializer.getCurrentBuffer().recycleBuffer();
				currentRecordDeserializer = null;
			}

			if (result.isFullRecord()) {
				processElement(deserializationDelegate.getInstance(), output);
				return InputStatus.MORE_AVAILABLE;
			}
		}

		Optional<BufferOrEvent> bufferOrEvent = checkpointedInputGate.pollNext();
		if (bufferOrEvent.isPresent()) {
			// return to the mailbox after receiving a checkpoint barrier to avoid processing of
			// data after the barrier before checkpoint is performed for unaligned checkpoint mode
			if (bufferOrEvent.get().isEvent() && bufferOrEvent.get().getEvent() instanceof CheckpointBarrier) {
				return InputStatus.MORE_AVAILABLE;
			}
			processBufferOrEvent(bufferOrEvent.get());
		} else {
			if (checkpointedInputGate.isFinished()) {
				checkState(checkpointedInputGate.getAvailableFuture().isDone(), "Finished BarrierHandler should be available");
				return InputStatus.END_OF_INPUT;
			}
			return InputStatus.NOTHING_AVAILABLE;
		}
	}
}
 

protected boolean getNextRecord(T target) throws IOException, InterruptedException {
	if (isFinished) {
		return false;
	}

	while (true) {
		if (currentRecordDeserializer != null) {
			DeserializationResult result = currentRecordDeserializer.getNextRecord(target);

			if (result.isBufferConsumed()) {
				final Buffer currentBuffer = currentRecordDeserializer.getCurrentBuffer();

				currentBuffer.recycleBuffer();
				currentRecordDeserializer = null;
			}

			if (result.isFullRecord()) {
				return true;
			}
		}

		final BufferOrEvent bufferOrEvent = inputGate.getNextBufferOrEvent().orElseThrow(IllegalStateException::new);

		if (bufferOrEvent.isBuffer()) {
			currentRecordDeserializer = recordDeserializers[bufferOrEvent.getChannelIndex()];
			currentRecordDeserializer.setNextBuffer(bufferOrEvent.getBuffer());
		}
		else {
			// sanity check for leftover data in deserializers. events should only come between
			// records, not in the middle of a fragment
			if (recordDeserializers[bufferOrEvent.getChannelIndex()].hasUnfinishedData()) {
				throw new IOException(
						"Received an event in channel " + bufferOrEvent.getChannelIndex() + " while still having "
						+ "data from a record. This indicates broken serialization logic. "
						+ "If you are using custom serialization code (Writable or Value types), check their "
						+ "serialization routines. In the case of Kryo, check the respective Kryo serializer.");
			}

			if (handleEvent(bufferOrEvent.getEvent())) {
				if (inputGate.isFinished()) {
					isFinished = true;
					return false;
				}
				else if (hasReachedEndOfSuperstep()) {
					return false;
				}
				// else: More data is coming...
			}
		}
	}
}
 

protected boolean getNextRecord(T target) throws IOException, InterruptedException {
	if (isFinished) {
		return false;
	}

	while (true) {
		if (currentRecordDeserializer != null) {
			DeserializationResult result = currentRecordDeserializer.getNextRecord(target);

			if (result.isBufferConsumed()) {
				final Buffer currentBuffer = currentRecordDeserializer.getCurrentBuffer();

				currentBuffer.recycleBuffer();
				currentRecordDeserializer = null;
			}

			if (result.isFullRecord()) {
				return true;
			}
		}

		final BufferOrEvent bufferOrEvent = inputGate.getNext().orElseThrow(IllegalStateException::new);

		if (bufferOrEvent.isBuffer()) {
			currentRecordDeserializer = recordDeserializers[bufferOrEvent.getChannelIndex()];
			currentRecordDeserializer.setNextBuffer(bufferOrEvent.getBuffer());
		}
		else {
			// sanity check for leftover data in deserializers. events should only come between
			// records, not in the middle of a fragment
			if (recordDeserializers[bufferOrEvent.getChannelIndex()].hasUnfinishedData()) {
				throw new IOException(
						"Received an event in channel " + bufferOrEvent.getChannelIndex() + " while still having "
						+ "data from a record. This indicates broken serialization logic. "
						+ "If you are using custom serialization code (Writable or Value types), check their "
						+ "serialization routines. In the case of Kryo, check the respective Kryo serializer.");
			}

			if (handleEvent(bufferOrEvent.getEvent())) {
				if (inputGate.isFinished()) {
					isFinished = true;
					return false;
				}
				else if (hasReachedEndOfSuperstep()) {
					return false;
				}
				// else: More data is coming...
			}
		}
	}
}
 

protected boolean getNextRecord(T target) throws IOException, InterruptedException {
	// The action of partition request was removed from InputGate#setup since FLINK-16536, and this is the only
	// unified way for launching partition request for batch jobs. In order to avoid potential performance concern,
	// we might consider migrating this action back to the setup based on some condition judgement future.
	if (!requestedPartitions) {
		inputGate.requestPartitions();
		requestedPartitions = true;
	}

	if (isFinished) {
		return false;
	}

	while (true) {
		if (currentRecordDeserializer != null) {
			DeserializationResult result = currentRecordDeserializer.getNextRecord(target);

			if (result.isBufferConsumed()) {
				final Buffer currentBuffer = currentRecordDeserializer.getCurrentBuffer();

				currentBuffer.recycleBuffer();
				currentRecordDeserializer = null;
			}

			if (result.isFullRecord()) {
				return true;
			}
		}

		final BufferOrEvent bufferOrEvent = inputGate.getNext().orElseThrow(IllegalStateException::new);

		if (bufferOrEvent.isBuffer()) {
			currentRecordDeserializer = recordDeserializers.get(bufferOrEvent.getChannelInfo());
			currentRecordDeserializer.setNextBuffer(bufferOrEvent.getBuffer());
		}
		else {
			// sanity check for leftover data in deserializers. events should only come between
			// records, not in the middle of a fragment
			if (recordDeserializers.get(bufferOrEvent.getChannelInfo()).hasUnfinishedData()) {
				throw new IOException(
						"Received an event in channel " + bufferOrEvent.getChannelInfo() + " while still having "
						+ "data from a record. This indicates broken serialization logic. "
						+ "If you are using custom serialization code (Writable or Value types), check their "
						+ "serialization routines. In the case of Kryo, check the respective Kryo serializer.");
			}

			if (handleEvent(bufferOrEvent.getEvent())) {
				if (inputGate.isFinished()) {
					isFinished = true;
					return false;
				}
				else if (hasReachedEndOfSuperstep()) {
					return false;
				}
				// else: More data is coming...
			}
		}
	}
}