org.apache.flink.runtime.io.network.NetworkSequenceViewReader Java Examples

@Override
public void userEventTriggered(ChannelHandlerContext ctx, Object msg) throws Exception {
	// The user event triggered event loop callback is used for thread-safe
	// hand over of reader queues and cancelled producers.

	if (msg instanceof NetworkSequenceViewReader) {
		enqueueAvailableReader((NetworkSequenceViewReader) msg);
	} else if (msg.getClass() == InputChannelID.class) {
		// Release partition view that get a cancel request.
		InputChannelID toCancel = (InputChannelID) msg;

		// remove reader from queue of available readers
		availableReaders.removeIf(reader -> reader.getReceiverId().equals(toCancel));

		// remove reader from queue of all readers and release its resource
		final NetworkSequenceViewReader toRelease = allReaders.remove(toCancel);
		if (toRelease != null) {
			releaseViewReader(toRelease);
		}
	} else {
		ctx.fireUserEventTriggered(msg);
	}
}
 

/**
 * Adds unannounced credits from the consumer or resumes data consumption after an exactly-once
 * checkpoint and enqueues the corresponding reader for this consumer (if not enqueued yet).
 *
 * @param receiverId The input channel id to identify the consumer.
 * @param operation The operation to be performed (add credit or resume data consumption).
 */
void addCreditOrResumeConsumption(
		InputChannelID receiverId,
		Consumer<NetworkSequenceViewReader> operation) throws Exception {
	if (fatalError) {
		return;
	}

	NetworkSequenceViewReader reader = allReaders.get(receiverId);
	if (reader != null) {
		operation.accept(reader);

		enqueueAvailableReader(reader);
	} else {
		throw new IllegalStateException("No reader for receiverId = " + receiverId + " exists.");
	}
}
 

@Override
public void userEventTriggered(ChannelHandlerContext ctx, Object msg) throws Exception {
	// The user event triggered event loop callback is used for thread-safe
	// hand over of reader queues and cancelled producers.

	if (msg instanceof NetworkSequenceViewReader) {
		enqueueAvailableReader((NetworkSequenceViewReader) msg);
	} else if (msg.getClass() == InputChannelID.class) {
		// Release partition view that get a cancel request.
		InputChannelID toCancel = (InputChannelID) msg;

		// remove reader from queue of available readers
		availableReaders.removeIf(reader -> reader.getReceiverId().equals(toCancel));

		// remove reader from queue of all readers and release its resource
		final NetworkSequenceViewReader toRelease = allReaders.remove(toCancel);
		if (toRelease != null) {
			releaseViewReader(toRelease);
		}
	} else {
		ctx.fireUserEventTriggered(msg);
	}
}
 

/**
 * Tests {@link PartitionRequestQueue#enqueueAvailableReader(NetworkSequenceViewReader)},
 * verifying the reader would be enqueued in the pipeline after resuming data consumption if there
 * are credit and data available.
 */
@Test
public void testEnqueueReaderByResumingConsumption() throws Exception {
	PipelinedSubpartition subpartition = PipelinedSubpartitionTest.createPipelinedSubpartition();
	Buffer.DataType dataType1 = Buffer.DataType.ALIGNED_EXACTLY_ONCE_CHECKPOINT_BARRIER;
	Buffer.DataType dataType2 = Buffer.DataType.DATA_BUFFER;
	subpartition.add(createEventBufferConsumer(4096, dataType1));
	subpartition.add(createEventBufferConsumer(4096, dataType2));

	BufferAvailabilityListener bufferAvailabilityListener = new NoOpBufferAvailablityListener();
	PipelinedSubpartitionView view = subpartition.createReadView(bufferAvailabilityListener);
	ResultPartitionProvider partitionProvider = (partitionId, index, availabilityListener) -> view;

	InputChannelID receiverId = new InputChannelID();
	PartitionRequestQueue queue = new PartitionRequestQueue();
	CreditBasedSequenceNumberingViewReader reader = new CreditBasedSequenceNumberingViewReader(receiverId, 0, queue);
	EmbeddedChannel channel = new EmbeddedChannel(queue);

	reader.requestSubpartitionView(partitionProvider, new ResultPartitionID(), 0);
	queue.notifyReaderCreated(reader);
	// we have adequate credits
	reader.addCredit(Integer.MAX_VALUE);
	assertTrue(reader.isAvailable());

	reader.notifyDataAvailable();
	channel.runPendingTasks();
	assertFalse(reader.isAvailable());
	assertEquals(1, subpartition.unsynchronizedGetNumberOfQueuedBuffers());

	queue.addCreditOrResumeConsumption(receiverId, NetworkSequenceViewReader::resumeConsumption);
	assertFalse(reader.isAvailable());
	assertEquals(0, subpartition.unsynchronizedGetNumberOfQueuedBuffers());

	Object data1 = channel.readOutbound();
	assertEquals(dataType1, ((NettyMessage.BufferResponse) data1).buffer.getDataType());
	Object data2 = channel.readOutbound();
	assertEquals(dataType2, ((NettyMessage.BufferResponse) data2).buffer.getDataType());
}
 

void notifyReaderNonEmpty(final NetworkSequenceViewReader reader) {
	// The notification might come from the same thread. For the initial writes this
	// might happen before the reader has set its reference to the view, because
	// creating the queue and the initial notification happen in the same method call.
	// This can be resolved by separating the creation of the view and allowing
	// notifications.

	// TODO This could potentially have a bad performance impact as in the
	// worst case (network consumes faster than the producer) each buffer
	// will trigger a separate event loop task being scheduled.
	ctx.executor().execute(() -> ctx.pipeline().fireUserEventTriggered(reader));
}
 

private void releaseAllResources() throws IOException {
	// note: this is only ever executed by one thread: the Netty IO thread!
	for (NetworkSequenceViewReader reader : allReaders.values()) {
		releaseViewReader(reader);
	}

	availableReaders.clear();
	allReaders.clear();
}
 

@Nullable
private NetworkSequenceViewReader pollAvailableReader() {
	NetworkSequenceViewReader reader = availableReaders.poll();
	if (reader != null) {
		reader.setRegisteredAsAvailable(false);
	}
	return reader;
}
 

public void close() throws IOException {
	if (ctx != null) {
		ctx.channel().close();
	}

	for (NetworkSequenceViewReader reader : allReaders.values()) {
		releaseViewReader(reader);
	}
	allReaders.clear();
}
 

/**
 * Try to enqueue the reader once receiving credit notification from the consumer or receiving
 * non-empty reader notification from the producer.
 *
 * <p>NOTE: Only one thread would trigger the actual enqueue after checking the reader's
 * availability, so there is no race condition here.
 */
private void enqueueAvailableReader(final NetworkSequenceViewReader reader) throws Exception {
	if (reader.isRegisteredAsAvailable() || !reader.isAvailable()) {
		return;
	}
	// Queue an available reader for consumption. If the queue is empty,
	// we try trigger the actual write. Otherwise this will be handled by
	// the writeAndFlushNextMessageIfPossible calls.
	boolean triggerWrite = availableReaders.isEmpty();
	registerAvailableReader(reader);

	if (triggerWrite) {
		writeAndFlushNextMessageIfPossible(ctx.channel());
	}
}
 

void notifyReaderNonEmpty(final NetworkSequenceViewReader reader) {
	// The notification might come from the same thread. For the initial writes this
	// might happen before the reader has set its reference to the view, because
	// creating the queue and the initial notification happen in the same method call.
	// This can be resolved by separating the creation of the view and allowing
	// notifications.

	// TODO This could potentially have a bad performance impact as in the
	// worst case (network consumes faster than the producer) each buffer
	// will trigger a separate event loop task being scheduled.
	ctx.executor().execute(() -> ctx.pipeline().fireUserEventTriggered(reader));
}
 

private void releaseAllResources() throws IOException {
	// note: this is only ever executed by one thread: the Netty IO thread!
	for (NetworkSequenceViewReader reader : allReaders.values()) {
		releaseViewReader(reader);
	}

	availableReaders.clear();
	allReaders.clear();
}
 

@Nullable
private NetworkSequenceViewReader pollAvailableReader() {
	NetworkSequenceViewReader reader = availableReaders.poll();
	if (reader != null) {
		reader.setRegisteredAsAvailable(false);
	}
	return reader;
}
 

/**
 * Adds unannounced credits from the consumer and enqueues the corresponding reader for this
 * consumer (if not enqueued yet).
 *
 * @param receiverId The input channel id to identify the consumer.
 * @param credit The unannounced credits of the consumer.
 */
void addCredit(InputChannelID receiverId, int credit) throws Exception {
	if (fatalError) {
		return;
	}

	NetworkSequenceViewReader reader = allReaders.get(receiverId);
	if (reader != null) {
		reader.addCredit(credit);

		enqueueAvailableReader(reader);
	} else {
		throw new IllegalStateException("No reader for receiverId = " + receiverId + " exists.");
	}
}
 

public void close() throws IOException {
	if (ctx != null) {
		ctx.channel().close();
	}

	for (NetworkSequenceViewReader reader : allReaders.values()) {
		releaseViewReader(reader);
	}
	allReaders.clear();
}
 

/**
 * Try to enqueue the reader once receiving credit notification from the consumer or receiving
 * non-empty reader notification from the producer.
 *
 * <p>NOTE: Only one thread would trigger the actual enqueue after checking the reader's
 * availability, so there is no race condition here.
 */
private void enqueueAvailableReader(final NetworkSequenceViewReader reader) throws Exception {
	if (reader.isRegisteredAsAvailable() || !reader.isAvailable()) {
		return;
	}
	// Queue an available reader for consumption. If the queue is empty,
	// we try trigger the actual write. Otherwise this will be handled by
	// the writeAndFlushNextMessageIfPossible calls.
	boolean triggerWrite = availableReaders.isEmpty();
	registerAvailableReader(reader);

	if (triggerWrite) {
		writeAndFlushNextMessageIfPossible(ctx.channel());
	}
}
 

/**
 * Try to enqueue the reader once receiving credit notification from the consumer or receiving
 * non-empty reader notification from the producer.
 *
 * <p>NOTE: Only one thread would trigger the actual enqueue after checking the reader's
 * availability, so there is no race condition here.
 */
private void enqueueAvailableReader(final NetworkSequenceViewReader reader) throws Exception {
	if (reader.isRegisteredAsAvailable() || !reader.isAvailable()) {
		return;
	}
	// Queue an available reader for consumption. If the queue is empty,
	// we try trigger the actual write. Otherwise this will be handled by
	// the writeAndFlushNextMessageIfPossible calls.
	boolean triggerWrite = availableReaders.isEmpty();
	registerAvailableReader(reader);

	if (triggerWrite) {
		writeAndFlushNextMessageIfPossible(ctx.channel());
	}
}
 

void notifyReaderNonEmpty(final NetworkSequenceViewReader reader) {
	// The notification might come from the same thread. For the initial writes this
	// might happen before the reader has set its reference to the view, because
	// creating the queue and the initial notification happen in the same method call.
	// This can be resolved by separating the creation of the view and allowing
	// notifications.

	// TODO This could potentially have a bad performance impact as in the
	// worst case (network consumes faster than the producer) each buffer
	// will trigger a separate event loop task being scheduled.
	ctx.executor().execute(() -> ctx.pipeline().fireUserEventTriggered(reader));
}
 

private void releaseAllResources() throws IOException {
	// note: this is only ever executed by one thread: the Netty IO thread!
	for (NetworkSequenceViewReader reader : allReaders.values()) {
		reader.releaseAllResources();
		markAsReleased(reader.getReceiverId());
	}

	availableReaders.clear();
	allReaders.clear();
}
 

@Nullable
private NetworkSequenceViewReader pollAvailableReader() {
	NetworkSequenceViewReader reader = availableReaders.poll();
	if (reader != null) {
		reader.setRegisteredAsAvailable(false);
	}
	return reader;
}
 

public void close() throws IOException {
	if (ctx != null) {
		ctx.channel().close();
	}

	for (NetworkSequenceViewReader reader : allReaders.values()) {
		reader.notifySubpartitionConsumed();
		reader.releaseAllResources();
		markAsReleased(reader.getReceiverId());
	}
	allReaders.clear();
}
 

/**
 * Adds unannounced credits from the consumer and enqueues the corresponding reader for this
 * consumer (if not enqueued yet).
 *
 * @param receiverId The input channel id to identify the consumer.
 * @param credit The unannounced credits of the consumer.
 */
void addCredit(InputChannelID receiverId, int credit) throws Exception {
	if (fatalError) {
		return;
	}

	NetworkSequenceViewReader reader = allReaders.get(receiverId);
	if (reader != null) {
		reader.addCredit(credit);

		enqueueAvailableReader(reader);
	} else {
		throw new IllegalStateException("No reader for receiverId = " + receiverId + " exists.");
	}
}
 

@Override
public void userEventTriggered(ChannelHandlerContext ctx, Object msg) throws Exception {
	// The user event triggered event loop callback is used for thread-safe
	// hand over of reader queues and cancelled producers.

	if (msg instanceof NetworkSequenceViewReader) {
		enqueueAvailableReader((NetworkSequenceViewReader) msg);
	} else if (msg.getClass() == InputChannelID.class) {
		// Release partition view that get a cancel request.
		InputChannelID toCancel = (InputChannelID) msg;
		if (released.contains(toCancel)) {
			return;
		}

		// Cancel the request for the input channel
		int size = availableReaders.size();
		for (int i = 0; i < size; i++) {
			NetworkSequenceViewReader reader = pollAvailableReader();
			if (reader.getReceiverId().equals(toCancel)) {
				reader.releaseAllResources();
				markAsReleased(reader.getReceiverId());
			} else {
				registerAvailableReader(reader);
			}
		}

		allReaders.remove(toCancel);
	} else {
		ctx.fireUserEventTriggered(msg);
	}
}
 

private void releaseViewReader(NetworkSequenceViewReader reader) throws IOException {
	reader.setRegisteredAsAvailable(false);
	reader.releaseAllResources();
}
 

public void notifyReaderCreated(final NetworkSequenceViewReader reader) {
	allReaders.put(reader.getReceiverId(), reader);
}
 

private void registerAvailableReader(NetworkSequenceViewReader reader) {
	availableReaders.add(reader);
	reader.setRegisteredAsAvailable(true);
}
 

public void notifyReaderCreated(final NetworkSequenceViewReader reader) {
	allReaders.put(reader.getReceiverId(), reader);
}
 

public void notifyReaderCreated(final NetworkSequenceViewReader reader) {
	allReaders.put(reader.getReceiverId(), reader);
}
 

private void registerAvailableReader(NetworkSequenceViewReader reader) {
	availableReaders.add(reader);
	reader.setRegisteredAsAvailable(true);
}
 

private void releaseViewReader(NetworkSequenceViewReader reader) throws IOException {
	reader.notifySubpartitionConsumed();
	reader.setRegisteredAsAvailable(false);
	reader.releaseAllResources();
}
 

private void registerAvailableReader(NetworkSequenceViewReader reader) {
	availableReaders.add(reader);
	reader.setRegisteredAsAvailable(true);
}