Java Code Examples for org.apache.flink.runtime.io.network.buffer.Buffer#setReaderIndex()

/**
 * Tests that broadcasted events' buffers are independent (in their (reader) indices) once they
 * are put into the queue for Netty when broadcasting events to multiple channels.
 */
@Test
public void testBroadcastEventBufferIndependence() throws Exception {
	@SuppressWarnings("unchecked")
	ArrayDeque<BufferConsumer>[] queues =
		new ArrayDeque[]{new ArrayDeque(), new ArrayDeque()};

	ResultPartitionWriter partition =
		new CollectingPartitionWriter(queues, new TestPooledBufferProvider(Integer.MAX_VALUE));
	RecordWriter<?> writer = new RecordWriter<>(partition);

	writer.broadcastEvent(EndOfPartitionEvent.INSTANCE);

	// Verify added to all queues
	assertEquals(1, queues[0].size());
	assertEquals(1, queues[1].size());

	// these two buffers may share the memory but not the indices!
	Buffer buffer1 = buildSingleBuffer(queues[0].remove());
	Buffer buffer2 = buildSingleBuffer(queues[1].remove());
	assertEquals(0, buffer1.getReaderIndex());
	assertEquals(0, buffer2.getReaderIndex());
	buffer1.setReaderIndex(1);
	assertEquals("Buffer 2 shares the same reader index as buffer 1", 0, buffer2.getReaderIndex());
}
 

/**
 * Tests that broadcasted records' buffers are independent (in their (reader) indices) once they
 * are put into the queue for Netty when broadcasting events to multiple channels.
 */
@Test
public void testBroadcastEmitBufferIndependence() throws Exception {
	@SuppressWarnings("unchecked")
	ArrayDeque<BufferConsumer>[] queues =
		new ArrayDeque[]{new ArrayDeque(), new ArrayDeque()};

	ResultPartitionWriter partition =
		new CollectingPartitionWriter(queues, new TestPooledBufferProvider(Integer.MAX_VALUE));
	RecordWriter<IntValue> writer = new RecordWriter<>(partition);

	writer.broadcastEmit(new IntValue(0));
	writer.flushAll();

	// Verify added to all queues
	assertEquals(1, queues[0].size());
	assertEquals(1, queues[1].size());

	// these two buffers may share the memory but not the indices!
	Buffer buffer1 = buildSingleBuffer(queues[0].remove());
	Buffer buffer2 = buildSingleBuffer(queues[1].remove());
	assertEquals(0, buffer1.getReaderIndex());
	assertEquals(0, buffer2.getReaderIndex());
	buffer1.setReaderIndex(1);
	assertEquals("Buffer 2 shares the same reader index as buffer 1", 0, buffer2.getReaderIndex());
}
 

private void verifyBroadcastBufferOrEventIndependence(boolean broadcastEvent) throws Exception {
	@SuppressWarnings("unchecked")
	ArrayDeque<BufferConsumer>[] queues = new ArrayDeque[]{new ArrayDeque(), new ArrayDeque()};

	ResultPartitionWriter partition =
		new CollectingPartitionWriter(queues, new TestPooledBufferProvider(Integer.MAX_VALUE));
	RecordWriter<IntValue> writer = new RecordWriterBuilder().build(partition);

	if (broadcastEvent) {
		writer.broadcastEvent(EndOfPartitionEvent.INSTANCE);
	} else {
		writer.broadcastEmit(new IntValue(0));
	}

	// verify added to all queues
	assertEquals(1, queues[0].size());
	assertEquals(1, queues[1].size());

	// these two buffers may share the memory but not the indices!
	Buffer buffer1 = buildSingleBuffer(queues[0].remove());
	Buffer buffer2 = buildSingleBuffer(queues[1].remove());
	assertEquals(0, buffer1.getReaderIndex());
	assertEquals(0, buffer2.getReaderIndex());
	buffer1.setReaderIndex(1);
	assertEquals("Buffer 2 shares the same reader index as buffer 1", 0, buffer2.getReaderIndex());
}
 

private void verifyBroadcastBufferOrEventIndependence(boolean broadcastEvent) throws Exception {
	@SuppressWarnings("unchecked")
	ArrayDeque<BufferConsumer>[] queues = new ArrayDeque[]{new ArrayDeque(), new ArrayDeque()};

	ResultPartitionWriter partition =
		new CollectingPartitionWriter(queues, new TestPooledBufferProvider(Integer.MAX_VALUE));
	RecordWriter<IntValue> writer = createRecordWriter(partition);

	if (broadcastEvent) {
		writer.broadcastEvent(EndOfPartitionEvent.INSTANCE);
	} else {
		writer.broadcastEmit(new IntValue(0));
	}

	// verify added to all queues
	assertEquals(1, queues[0].size());
	assertEquals(1, queues[1].size());

	// these two buffers may share the memory but not the indices!
	Buffer buffer1 = buildSingleBuffer(queues[0].remove());
	Buffer buffer2 = buildSingleBuffer(queues[1].remove());
	assertEquals(0, buffer1.getReaderIndex());
	assertEquals(0, buffer2.getReaderIndex());
	buffer1.setReaderIndex(1);
	assertEquals("Buffer 2 shares the same reader index as buffer 1", 0, buffer2.getReaderIndex());
}
 

private boolean isEndOfChannelStateEvent(Buffer buffer) throws IOException {
	if (buffer.isBuffer()) {
		return false;
	}

	AbstractEvent event = EventSerializer.fromBuffer(buffer, getClass().getClassLoader());
	buffer.setReaderIndex(0);
	return event.getClass() == EndOfChannelStateEvent.class;
}
 

/**
 * Parses the buffer as an event and returns the {@link CheckpointBarrier} if the event is indeed a barrier or
 * returns null in all other cases.
 */
@Nullable
protected CheckpointBarrier parseCheckpointBarrierOrNull(Buffer buffer) throws IOException {
	if (buffer.isBuffer()) {
		return null;
	}

	AbstractEvent event = EventSerializer.fromBuffer(buffer, getClass().getClassLoader());
	// reset the buffer because it would be deserialized again in SingleInputGate while getting next buffer.
	// we can further improve to avoid double deserialization in the future.
	buffer.setReaderIndex(0);
	return event.getClass() == CheckpointBarrier.class ? (CheckpointBarrier) event : null;
}