org.apache.flink.runtime.io.network.netty.NettyMessage.BufferResponse Java Examples

The following examples show how to use org.apache.flink.runtime.io.network.netty.NettyMessage.BufferResponse. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: CreditBasedPartitionRequestClientHandlerTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests a fix for FLINK-1627.
 *
 * <p> FLINK-1627 discovered a race condition, which could lead to an infinite loop when a
 * receiver was cancelled during a certain time of decoding a message. The test reproduces the
 * input, which lead to the infinite loop: when the handler gets a reference to the buffer
 * provider of the receiving input channel, but the respective input channel is released (and
 * the corresponding buffer provider destroyed), the handler did not notice this.
 *
 * @see <a href="https://issues.apache.org/jira/browse/FLINK-1627">FLINK-1627</a>
 */
@Test(timeout = 60000)
@SuppressWarnings("unchecked")
public void testReleaseInputChannelDuringDecode() throws Exception {
	// Mocks an input channel in a state as it was released during a decode.
	final BufferProvider bufferProvider = mock(BufferProvider.class);
	when(bufferProvider.requestBuffer()).thenReturn(null);
	when(bufferProvider.isDestroyed()).thenReturn(true);
	when(bufferProvider.addBufferListener(any(BufferListener.class))).thenReturn(false);

	final RemoteInputChannel inputChannel = mock(RemoteInputChannel.class);
	when(inputChannel.getInputChannelId()).thenReturn(new InputChannelID());
	when(inputChannel.getBufferProvider()).thenReturn(bufferProvider);

	final BufferResponse receivedBuffer = createBufferResponse(
			TestBufferFactory.createBuffer(TestBufferFactory.BUFFER_SIZE), 0, inputChannel.getInputChannelId(), 2);

	final CreditBasedPartitionRequestClientHandler client = new CreditBasedPartitionRequestClientHandler();
	client.addInputChannel(inputChannel);

	client.channelRead(mock(ChannelHandlerContext.class), receivedBuffer);
}
 
Example #2
Source File: CreditBasedPartitionRequestClientHandlerTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Returns a deserialized buffer message as it would be received during runtime.
 */
private static BufferResponse createBufferResponse(
		Buffer buffer,
		int sequenceNumber,
		InputChannelID receivingChannelId,
		int backlog,
		NetworkBufferAllocator allocator) throws IOException {
	// Mock buffer to serialize
	BufferResponse resp = new BufferResponse(buffer, sequenceNumber, receivingChannelId, backlog);

	ByteBuf serialized = resp.write(UnpooledByteBufAllocator.DEFAULT);

	// Skip general header bytes
	serialized.readBytes(NettyMessage.FRAME_HEADER_LENGTH);

	// Deserialize the bytes to construct the BufferResponse.
	return BufferResponse.readFrom(serialized, allocator);
}
 
Example #3
Source File: CreditBasedPartitionRequestClientHandlerTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Verifies that {@link RemoteInputChannel#onError(Throwable)} is called when a
 * {@link BufferResponse} is received but no available buffer in input channel.
 */
@Test
public void testThrowExceptionForNoAvailableBuffer() throws Exception {
	final SingleInputGate inputGate = createSingleInputGate(1);
	final RemoteInputChannel inputChannel = spy(InputChannelBuilder.newBuilder().buildRemoteChannel(inputGate));

	final CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
	handler.addInputChannel(inputChannel);

	assertEquals("There should be no buffers available in the channel.",
			0, inputChannel.getNumberOfAvailableBuffers());

	final BufferResponse bufferResponse = createBufferResponse(
		TestBufferFactory.createBuffer(TestBufferFactory.BUFFER_SIZE),
		0,
		inputChannel.getInputChannelId(),
		2,
		new NetworkBufferAllocator(handler));
	assertNull(bufferResponse.getBuffer());

	handler.channelRead(mock(ChannelHandlerContext.class), bufferResponse);
	verify(inputChannel, times(1)).onError(any(IllegalStateException.class));
}
 
Example #4
Source File: CreditBasedPartitionRequestClientHandlerTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests a fix for FLINK-1627.
 *
 * <p> FLINK-1627 discovered a race condition, which could lead to an infinite loop when a
 * receiver was cancelled during a certain time of decoding a message. The test reproduces the
 * input, which lead to the infinite loop: when the handler gets a reference to the buffer
 * provider of the receiving input channel, but the respective input channel is released (and
 * the corresponding buffer provider destroyed), the handler did not notice this.
 *
 * @see <a href="https://issues.apache.org/jira/browse/FLINK-1627">FLINK-1627</a>
 */
@Test(timeout = 60000)
@SuppressWarnings("unchecked")
public void testReleaseInputChannelDuringDecode() throws Exception {
	// Mocks an input channel in a state as it was released during a decode.
	final BufferProvider bufferProvider = mock(BufferProvider.class);
	when(bufferProvider.requestBuffer()).thenReturn(null);
	when(bufferProvider.isDestroyed()).thenReturn(true);
	when(bufferProvider.addBufferListener(any(BufferListener.class))).thenReturn(false);

	final RemoteInputChannel inputChannel = mock(RemoteInputChannel.class);
	when(inputChannel.getInputChannelId()).thenReturn(new InputChannelID());
	when(inputChannel.getBufferProvider()).thenReturn(bufferProvider);

	final CreditBasedPartitionRequestClientHandler client = new CreditBasedPartitionRequestClientHandler();
	client.addInputChannel(inputChannel);

	final BufferResponse receivedBuffer = createBufferResponse(
		TestBufferFactory.createBuffer(TestBufferFactory.BUFFER_SIZE),
		0,
		inputChannel.getInputChannelId(),
		2,
		new NetworkBufferAllocator(client));

	client.channelRead(mock(ChannelHandlerContext.class), receivedBuffer);
}
 
Example #5
Source File: CreditBasedPartitionRequestClientHandlerTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Verifies that {@link RemoteInputChannel#onError(Throwable)} is called when a
 * {@link BufferResponse} is received but no available buffer in input channel.
 */
@Test
public void testThrowExceptionForNoAvailableBuffer() throws Exception {
	final SingleInputGate inputGate = createSingleInputGate(1);
	final RemoteInputChannel inputChannel = spy(InputChannelBuilder.newBuilder().buildRemoteAndSetToGate(inputGate));

	final CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
	handler.addInputChannel(inputChannel);

	assertEquals("There should be no buffers available in the channel.",
			0, inputChannel.getNumberOfAvailableBuffers());

	final BufferResponse bufferResponse = createBufferResponse(
		TestBufferFactory.createBuffer(TestBufferFactory.BUFFER_SIZE), 0, inputChannel.getInputChannelId(), 2);
	handler.channelRead(mock(ChannelHandlerContext.class), bufferResponse);

	verify(inputChannel, times(1)).onError(any(IllegalStateException.class));
}
 
Example #6
Source File: PartitionRequestClientHandlerTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Tests a fix for FLINK-1627.
 *
 * <p> FLINK-1627 discovered a race condition, which could lead to an infinite loop when a
 * receiver was cancelled during a certain time of decoding a message. The test reproduces the
 * input, which lead to the infinite loop: when the handler gets a reference to the buffer
 * provider of the receiving input channel, but the respective input channel is released (and
 * the corresponding buffer provider destroyed), the handler did not notice this.
 *
 * @see <a href="https://issues.apache.org/jira/browse/FLINK-1627">FLINK-1627</a>
 */
@Test(timeout = 60000)
@SuppressWarnings("unchecked")
public void testReleaseInputChannelDuringDecode() throws Exception {
	// Mocks an input channel in a state as it was released during a decode.
	final BufferProvider bufferProvider = mock(BufferProvider.class);
	when(bufferProvider.requestBuffer()).thenReturn(null);
	when(bufferProvider.isDestroyed()).thenReturn(true);
	when(bufferProvider.addBufferListener(any(BufferListener.class))).thenReturn(false);

	final RemoteInputChannel inputChannel = mock(RemoteInputChannel.class);
	when(inputChannel.getInputChannelId()).thenReturn(new InputChannelID());
	when(inputChannel.getBufferProvider()).thenReturn(bufferProvider);

	final BufferResponse receivedBuffer = createBufferResponse(
			TestBufferFactory.createBuffer(TestBufferFactory.BUFFER_SIZE), 0, inputChannel.getInputChannelId(), 2);

	final PartitionRequestClientHandler client = new PartitionRequestClientHandler();
	client.addInputChannel(inputChannel);

	client.channelRead(mock(ChannelHandlerContext.class), receivedBuffer);
}
 
Example #7
Source File: PartitionRequestClientHandlerTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Returns a deserialized buffer message as it would be received during runtime.
 */
static BufferResponse createBufferResponse(
		Buffer buffer,
		int sequenceNumber,
		InputChannelID receivingChannelId,
		int backlog) throws IOException {

	// Mock buffer to serialize
	BufferResponse resp = new BufferResponse(buffer, sequenceNumber, receivingChannelId, backlog);

	ByteBuf serialized = resp.write(UnpooledByteBufAllocator.DEFAULT);

	// Skip general header bytes
	serialized.readBytes(NettyMessage.FRAME_HEADER_LENGTH);

	// Deserialize the bytes again. We have to go this way, because we only partly deserialize
	// the header of the response and wait for a buffer from the buffer pool to copy the payload
	// data into.
	BufferResponse deserialized = BufferResponse.readFrom(serialized);

	return deserialized;
}
 
Example #8
Source File: PartitionRequestClientHandlerTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests a fix for FLINK-1627.
 *
 * <p> FLINK-1627 discovered a race condition, which could lead to an infinite loop when a
 * receiver was cancelled during a certain time of decoding a message. The test reproduces the
 * input, which lead to the infinite loop: when the handler gets a reference to the buffer
 * provider of the receiving input channel, but the respective input channel is released (and
 * the corresponding buffer provider destroyed), the handler did not notice this.
 *
 * @see <a href="https://issues.apache.org/jira/browse/FLINK-1627">FLINK-1627</a>
 */
@Test(timeout = 60000)
@SuppressWarnings("unchecked")
public void testReleaseInputChannelDuringDecode() throws Exception {
	// Mocks an input channel in a state as it was released during a decode.
	final BufferProvider bufferProvider = mock(BufferProvider.class);
	when(bufferProvider.requestBuffer()).thenReturn(null);
	when(bufferProvider.isDestroyed()).thenReturn(true);
	when(bufferProvider.addBufferListener(any(BufferListener.class))).thenReturn(false);

	final RemoteInputChannel inputChannel = mock(RemoteInputChannel.class);
	when(inputChannel.getInputChannelId()).thenReturn(new InputChannelID());
	when(inputChannel.getBufferProvider()).thenReturn(bufferProvider);

	final BufferResponse receivedBuffer = createBufferResponse(
			TestBufferFactory.createBuffer(TestBufferFactory.BUFFER_SIZE), 0, inputChannel.getInputChannelId(), 2);

	final PartitionRequestClientHandler client = new PartitionRequestClientHandler();
	client.addInputChannel(inputChannel);

	client.channelRead(mock(ChannelHandlerContext.class), receivedBuffer);
}
 
Example #9
Source File: CreditBasedPartitionRequestClientHandlerTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Verifies that {@link RemoteInputChannel#onError(Throwable)} is called when a
 * {@link BufferResponse} is received but no available buffer in input channel.
 */
@Test
public void testThrowExceptionForNoAvailableBuffer() throws Exception {
	final SingleInputGate inputGate = createSingleInputGate();
	final RemoteInputChannel inputChannel = spy(createRemoteInputChannel(inputGate));

	final CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
	handler.addInputChannel(inputChannel);

	assertEquals("There should be no buffers available in the channel.",
			0, inputChannel.getNumberOfAvailableBuffers());

	final BufferResponse bufferResponse = createBufferResponse(
		TestBufferFactory.createBuffer(TestBufferFactory.BUFFER_SIZE), 0, inputChannel.getInputChannelId(), 2);
	handler.channelRead(mock(ChannelHandlerContext.class), bufferResponse);

	verify(inputChannel, times(1)).onError(any(IllegalStateException.class));
}
 
Example #10
Source File: PartitionRequestClientHandlerTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Returns a deserialized buffer message as it would be received during runtime.
 */
static BufferResponse createBufferResponse(
		Buffer buffer,
		int sequenceNumber,
		InputChannelID receivingChannelId,
		int backlog) throws IOException {

	// Mock buffer to serialize
	BufferResponse resp = new BufferResponse(buffer, sequenceNumber, receivingChannelId, backlog);

	ByteBuf serialized = resp.write(UnpooledByteBufAllocator.DEFAULT);

	// Skip general header bytes
	serialized.readBytes(NettyMessage.FRAME_HEADER_LENGTH);

	// Deserialize the bytes again. We have to go this way, because we only partly deserialize
	// the header of the response and wait for a buffer from the buffer pool to copy the payload
	// data into.
	BufferResponse deserialized = BufferResponse.readFrom(serialized);

	return deserialized;
}
 
Example #11
Source File: CreditBasedPartitionRequestClientHandlerTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Tests a fix for FLINK-1627.
 *
 * <p> FLINK-1627 discovered a race condition, which could lead to an infinite loop when a
 * receiver was cancelled during a certain time of decoding a message. The test reproduces the
 * input, which lead to the infinite loop: when the handler gets a reference to the buffer
 * provider of the receiving input channel, but the respective input channel is released (and
 * the corresponding buffer provider destroyed), the handler did not notice this.
 *
 * @see <a href="https://issues.apache.org/jira/browse/FLINK-1627">FLINK-1627</a>
 */
@Test(timeout = 60000)
@SuppressWarnings("unchecked")
public void testReleaseInputChannelDuringDecode() throws Exception {
	// Mocks an input channel in a state as it was released during a decode.
	final BufferProvider bufferProvider = mock(BufferProvider.class);
	when(bufferProvider.requestBuffer()).thenReturn(null);
	when(bufferProvider.isDestroyed()).thenReturn(true);
	when(bufferProvider.addBufferListener(any(BufferListener.class))).thenReturn(false);

	final RemoteInputChannel inputChannel = mock(RemoteInputChannel.class);
	when(inputChannel.getInputChannelId()).thenReturn(new InputChannelID());
	when(inputChannel.getBufferProvider()).thenReturn(bufferProvider);

	final BufferResponse receivedBuffer = createBufferResponse(
			TestBufferFactory.createBuffer(TestBufferFactory.BUFFER_SIZE), 0, inputChannel.getInputChannelId(), 2);

	final CreditBasedPartitionRequestClientHandler client = new CreditBasedPartitionRequestClientHandler();
	client.addInputChannel(inputChannel);

	client.channelRead(mock(ChannelHandlerContext.class), receivedBuffer);
}
 
Example #12
Source File: CreditBasedPartitionRequestClientHandlerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Verifies that {@link RemoteInputChannel#onBuffer(Buffer, int, int)} is called when a
 * {@link BufferResponse} is received.
 */
@Test
public void testReceiveBuffer() throws Exception {
	final NetworkBufferPool networkBufferPool = new NetworkBufferPool(10, 32, 2);
	final SingleInputGate inputGate = createSingleInputGate(1);
	final RemoteInputChannel inputChannel = InputChannelBuilder.newBuilder()
		.setMemorySegmentProvider(networkBufferPool)
		.buildRemoteAndSetToGate(inputGate);
	try {
		final BufferPool bufferPool = networkBufferPool.createBufferPool(8, 8);
		inputGate.setBufferPool(bufferPool);
		inputGate.assignExclusiveSegments();

		final CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
		handler.addInputChannel(inputChannel);

		final int backlog = 2;
		final BufferResponse bufferResponse = createBufferResponse(
			TestBufferFactory.createBuffer(32), 0, inputChannel.getInputChannelId(), backlog);
		handler.channelRead(mock(ChannelHandlerContext.class), bufferResponse);

		assertEquals(1, inputChannel.getNumberOfQueuedBuffers());
		assertEquals(2, inputChannel.getSenderBacklog());
	} finally {
		// Release all the buffer resources
		inputGate.close();

		networkBufferPool.destroyAllBufferPools();
		networkBufferPool.destroy();
	}
}
 
Example #13
Source File: PartitionRequestClientHandlerTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Verifies that {@link RemoteInputChannel#onBuffer(Buffer, int, int)} is called when a
 * {@link BufferResponse} is received.
 */
@Test
public void testReceiveBuffer() throws Exception {
	final NetworkBufferPool networkBufferPool = new NetworkBufferPool(10, 32);
	final SingleInputGate inputGate = createSingleInputGate();
	final RemoteInputChannel inputChannel = createRemoteInputChannel(inputGate);
	try {
		final BufferPool bufferPool = networkBufferPool.createBufferPool(8, 8);
		inputGate.setBufferPool(bufferPool);
		final int numExclusiveBuffers = 2;
		inputGate.assignExclusiveSegments(networkBufferPool, numExclusiveBuffers);

		final PartitionRequestClientHandler handler = new PartitionRequestClientHandler();
		handler.addInputChannel(inputChannel);

		final int backlog = 2;
		final BufferResponse bufferResponse = createBufferResponse(
			TestBufferFactory.createBuffer(32), 0, inputChannel.getInputChannelId(), backlog);
		handler.channelRead(mock(ChannelHandlerContext.class), bufferResponse);

		assertEquals(1, inputChannel.getNumberOfQueuedBuffers());
	} finally {
		// Release all the buffer resources
		inputGate.releaseAllResources();

		networkBufferPool.destroyAllBufferPools();
		networkBufferPool.destroy();
	}
}
 
Example #14
Source File: CreditBasedPartitionRequestClientHandlerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Verifies that {@link RemoteInputChannel#onBuffer(Buffer, int, int)} is called when a
 * {@link BufferResponse} is received.
 */
@Test
public void testReceiveBuffer() throws Exception {
	final NetworkBufferPool networkBufferPool = new NetworkBufferPool(10, 32, 2);
	final SingleInputGate inputGate = createSingleInputGate(1, networkBufferPool);
	final RemoteInputChannel inputChannel = InputChannelBuilder.newBuilder().buildRemoteChannel(inputGate);
	try {
		inputGate.setInputChannels(inputChannel);
		final BufferPool bufferPool = networkBufferPool.createBufferPool(8, 8);
		inputGate.setBufferPool(bufferPool);
		inputGate.assignExclusiveSegments();

		final CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
		handler.addInputChannel(inputChannel);

		final int backlog = 2;
		final BufferResponse bufferResponse = createBufferResponse(
			TestBufferFactory.createBuffer(32),
			0,
			inputChannel.getInputChannelId(),
			backlog,
			new NetworkBufferAllocator(handler));
		handler.channelRead(mock(ChannelHandlerContext.class), bufferResponse);

		assertEquals(1, inputChannel.getNumberOfQueuedBuffers());
		assertEquals(2, inputChannel.getSenderBacklog());
	} finally {
		releaseResource(inputGate, networkBufferPool);
	}
}
 
Example #15
Source File: CreditBasedPartitionRequestClientHandlerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests a fix for FLINK-1761.
 *
 * <p>FLINK-1761 discovered an IndexOutOfBoundsException, when receiving buffers of size 0.
 */
@Test
public void testReceiveEmptyBuffer() throws Exception {
	// Minimal mock of a remote input channel
	final BufferProvider bufferProvider = mock(BufferProvider.class);
	when(bufferProvider.requestBuffer()).thenReturn(TestBufferFactory.createBuffer(0));

	final RemoteInputChannel inputChannel = mock(RemoteInputChannel.class);
	when(inputChannel.getInputChannelId()).thenReturn(new InputChannelID());
	when(inputChannel.getBufferProvider()).thenReturn(bufferProvider);

	// An empty buffer of size 0
	final Buffer emptyBuffer = TestBufferFactory.createBuffer(0);

	final CreditBasedPartitionRequestClientHandler client = new CreditBasedPartitionRequestClientHandler();
	client.addInputChannel(inputChannel);

	final int backlog = 2;
	final BufferResponse receivedBuffer = createBufferResponse(
		emptyBuffer,
		0,
		inputChannel.getInputChannelId(),
		backlog,
		new NetworkBufferAllocator(client));

	// Read the empty buffer
	client.channelRead(mock(ChannelHandlerContext.class), receivedBuffer);

	// This should not throw an exception
	verify(inputChannel, never()).onError(any(Throwable.class));
	verify(inputChannel, times(1)).onEmptyBuffer(0, backlog);
}
 
Example #16
Source File: CreditBasedPartitionRequestClientHandlerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testDoNotFailHandlerOnSingleChannelFailure() throws Exception {
	// Setup
	final int bufferSize = 1024;
	final String expectedMessage = "test exception on buffer";
	final NetworkBufferPool networkBufferPool = new NetworkBufferPool(10, bufferSize, 2);
	final SingleInputGate inputGate = createSingleInputGate(1, networkBufferPool);
	final RemoteInputChannel inputChannel = new TestRemoteInputChannelForError(inputGate, expectedMessage);
	final CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();

	try {
		inputGate.setInputChannels(inputChannel);
		inputGate.assignExclusiveSegments();
		inputGate.requestPartitions();
		handler.addInputChannel(inputChannel);

		final BufferResponse bufferResponse = createBufferResponse(
			TestBufferFactory.createBuffer(bufferSize),
			0,
			inputChannel.getInputChannelId(),
			1,
			new NetworkBufferAllocator(handler));

		// It will trigger an expected exception from TestRemoteInputChannelForError#onBuffer
		handler.channelRead(null, bufferResponse);

		// The handler should not be tagged as error for above excepted exception
		handler.checkError();

		try {
			// The input channel should be tagged as error and the respective exception is thrown via #getNext
			inputGate.getNext();
		} catch (IOException ignored) {
			assertEquals(expectedMessage, ignored.getMessage());
		}
	} finally {
		// Cleanup
		releaseResource(inputGate, networkBufferPool);
	}
}
 
Example #17
Source File: CreditBasedPartitionRequestClientHandlerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests a fix for FLINK-1761.
 *
 * <p>FLINK-1761 discovered an IndexOutOfBoundsException, when receiving buffers of size 0.
 */
@Test
public void testReceiveEmptyBuffer() throws Exception {
	// Minimal mock of a remote input channel
	final BufferProvider bufferProvider = mock(BufferProvider.class);
	when(bufferProvider.requestBuffer()).thenReturn(TestBufferFactory.createBuffer(0));

	final RemoteInputChannel inputChannel = mock(RemoteInputChannel.class);
	when(inputChannel.getInputChannelId()).thenReturn(new InputChannelID());
	when(inputChannel.getBufferProvider()).thenReturn(bufferProvider);

	// An empty buffer of size 0
	final Buffer emptyBuffer = TestBufferFactory.createBuffer(0);

	final int backlog = 2;
	final BufferResponse receivedBuffer = createBufferResponse(
		emptyBuffer, 0, inputChannel.getInputChannelId(), backlog);

	final CreditBasedPartitionRequestClientHandler client = new CreditBasedPartitionRequestClientHandler();
	client.addInputChannel(inputChannel);

	// Read the empty buffer
	client.channelRead(mock(ChannelHandlerContext.class), receivedBuffer);

	// This should not throw an exception
	verify(inputChannel, never()).onError(any(Throwable.class));
	verify(inputChannel, times(1)).onEmptyBuffer(0, backlog);
}
 
Example #18
Source File: CreditBasedPartitionRequestClientHandlerTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Tests a fix for FLINK-1761.
 *
 * <p>FLINK-1761 discovered an IndexOutOfBoundsException, when receiving buffers of size 0.
 */
@Test
public void testReceiveEmptyBuffer() throws Exception {
	// Minimal mock of a remote input channel
	final BufferProvider bufferProvider = mock(BufferProvider.class);
	when(bufferProvider.requestBuffer()).thenReturn(TestBufferFactory.createBuffer(0));

	final RemoteInputChannel inputChannel = mock(RemoteInputChannel.class);
	when(inputChannel.getInputChannelId()).thenReturn(new InputChannelID());
	when(inputChannel.getBufferProvider()).thenReturn(bufferProvider);

	// An empty buffer of size 0
	final Buffer emptyBuffer = TestBufferFactory.createBuffer(0);

	final int backlog = 2;
	final BufferResponse receivedBuffer = createBufferResponse(
		emptyBuffer, 0, inputChannel.getInputChannelId(), backlog);

	final CreditBasedPartitionRequestClientHandler client = new CreditBasedPartitionRequestClientHandler();
	client.addInputChannel(inputChannel);

	// Read the empty buffer
	client.channelRead(mock(ChannelHandlerContext.class), receivedBuffer);

	// This should not throw an exception
	verify(inputChannel, never()).onError(any(Throwable.class));
	verify(inputChannel, times(1)).onEmptyBuffer(0, backlog);
}
 
Example #19
Source File: CreditBasedPartitionRequestClientHandlerTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Verifies that {@link RemoteInputChannel#onBuffer(Buffer, int, int)} is called when a
 * {@link BufferResponse} is received.
 */
@Test
public void testReceiveBuffer() throws Exception {
	final NetworkBufferPool networkBufferPool = new NetworkBufferPool(10, 32);
	final SingleInputGate inputGate = createSingleInputGate();
	final RemoteInputChannel inputChannel = createRemoteInputChannel(inputGate);
	try {
		final BufferPool bufferPool = networkBufferPool.createBufferPool(8, 8);
		inputGate.setBufferPool(bufferPool);
		final int numExclusiveBuffers = 2;
		inputGate.assignExclusiveSegments(networkBufferPool, numExclusiveBuffers);

		final CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
		handler.addInputChannel(inputChannel);

		final int backlog = 2;
		final BufferResponse bufferResponse = createBufferResponse(
			TestBufferFactory.createBuffer(32), 0, inputChannel.getInputChannelId(), backlog);
		handler.channelRead(mock(ChannelHandlerContext.class), bufferResponse);

		assertEquals(1, inputChannel.getNumberOfQueuedBuffers());
		assertEquals(2, inputChannel.getSenderBacklog());
	} finally {
		// Release all the buffer resources
		inputGate.releaseAllResources();

		networkBufferPool.destroyAllBufferPools();
		networkBufferPool.destroy();
	}
}
 
Example #20
Source File: PartitionRequestClientHandlerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Verifies that {@link RemoteInputChannel#onBuffer(Buffer, int, int)} is called when a
 * {@link BufferResponse} is received.
 */
@Test
public void testReceiveBuffer() throws Exception {
	final NetworkBufferPool networkBufferPool = new NetworkBufferPool(10, 32, 2);
	final SingleInputGate inputGate = createSingleInputGate(1);
	final RemoteInputChannel inputChannel = InputChannelBuilder.newBuilder()
		.setMemorySegmentProvider(networkBufferPool)
		.buildRemoteAndSetToGate(inputGate);
	try {
		final BufferPool bufferPool = networkBufferPool.createBufferPool(8, 8);
		inputGate.setBufferPool(bufferPool);
		inputGate.assignExclusiveSegments();

		final PartitionRequestClientHandler handler = new PartitionRequestClientHandler();
		handler.addInputChannel(inputChannel);

		final int backlog = 2;
		final BufferResponse bufferResponse = createBufferResponse(
			TestBufferFactory.createBuffer(32), 0, inputChannel.getInputChannelId(), backlog);
		handler.channelRead(mock(ChannelHandlerContext.class), bufferResponse);

		assertEquals(1, inputChannel.getNumberOfQueuedBuffers());
	} finally {
		// Release all the buffer resources
		inputGate.close();

		networkBufferPool.destroyAllBufferPools();
		networkBufferPool.destroy();
	}
}
 
Example #21
Source File: PartitionRequestClientHandlerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests a fix for FLINK-1761.
 *
 * <p>FLINK-1761 discovered an IndexOutOfBoundsException, when receiving buffers of size 0.
 */
@Test
public void testReceiveEmptyBuffer() throws Exception {
	// Minimal mock of a remote input channel
	final BufferProvider bufferProvider = mock(BufferProvider.class);
	when(bufferProvider.requestBuffer()).thenReturn(TestBufferFactory.createBuffer(0));

	final RemoteInputChannel inputChannel = mock(RemoteInputChannel.class);
	when(inputChannel.getInputChannelId()).thenReturn(new InputChannelID());
	when(inputChannel.getBufferProvider()).thenReturn(bufferProvider);

	// An empty buffer of size 0
	final Buffer emptyBuffer = TestBufferFactory.createBuffer(0);

	final int backlog = -1;
	final BufferResponse receivedBuffer = createBufferResponse(
		emptyBuffer, 0, inputChannel.getInputChannelId(), backlog);

	final PartitionRequestClientHandler client = new PartitionRequestClientHandler();
	client.addInputChannel(inputChannel);

	// Read the empty buffer
	client.channelRead(mock(ChannelHandlerContext.class), receivedBuffer);

	// This should not throw an exception
	verify(inputChannel, never()).onError(any(Throwable.class));
	verify(inputChannel, times(1)).onEmptyBuffer(0, backlog);
}
 
Example #22
Source File: PartitionRequestClientHandlerTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Tests a fix for FLINK-1761.
 *
 * <p>FLINK-1761 discovered an IndexOutOfBoundsException, when receiving buffers of size 0.
 */
@Test
public void testReceiveEmptyBuffer() throws Exception {
	// Minimal mock of a remote input channel
	final BufferProvider bufferProvider = mock(BufferProvider.class);
	when(bufferProvider.requestBuffer()).thenReturn(TestBufferFactory.createBuffer(0));

	final RemoteInputChannel inputChannel = mock(RemoteInputChannel.class);
	when(inputChannel.getInputChannelId()).thenReturn(new InputChannelID());
	when(inputChannel.getBufferProvider()).thenReturn(bufferProvider);

	// An empty buffer of size 0
	final Buffer emptyBuffer = TestBufferFactory.createBuffer(0);

	final int backlog = -1;
	final BufferResponse receivedBuffer = createBufferResponse(
		emptyBuffer, 0, inputChannel.getInputChannelId(), backlog);

	final PartitionRequestClientHandler client = new PartitionRequestClientHandler();
	client.addInputChannel(inputChannel);

	// Read the empty buffer
	client.channelRead(mock(ChannelHandlerContext.class), receivedBuffer);

	// This should not throw an exception
	verify(inputChannel, never()).onError(any(Throwable.class));
	verify(inputChannel, times(1)).onEmptyBuffer(0, backlog);
}
 
Example #23
Source File: CreditBasedPartitionRequestClientHandlerTest.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Verifies that {@link RemoteInputChannel} is enqueued in the pipeline for notifying credits,
 * and verifies the behaviour of credit notification by triggering channel's writability changed.
 */
@Test
public void testNotifyCreditAvailable() throws Exception {
	final CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
	final EmbeddedChannel channel = new EmbeddedChannel(handler);
	final PartitionRequestClient client = new NettyPartitionRequestClient(
		channel, handler, mock(ConnectionID.class), mock(PartitionRequestClientFactory.class));

	final NetworkBufferPool networkBufferPool = new NetworkBufferPool(10, 32, 2);
	final SingleInputGate inputGate = createSingleInputGate(1);
	final RemoteInputChannel inputChannel1 = createRemoteInputChannel(inputGate, client, networkBufferPool);
	final RemoteInputChannel inputChannel2 = createRemoteInputChannel(inputGate, client, networkBufferPool);
	try {
		final BufferPool bufferPool = networkBufferPool.createBufferPool(6, 6);
		inputGate.setBufferPool(bufferPool);
		inputGate.assignExclusiveSegments();

		inputChannel1.requestSubpartition(0);
		inputChannel2.requestSubpartition(0);

		// The two input channels should send partition requests
		assertTrue(channel.isWritable());
		Object readFromOutbound = channel.readOutbound();
		assertThat(readFromOutbound, instanceOf(PartitionRequest.class));
		assertEquals(inputChannel1.getInputChannelId(), ((PartitionRequest) readFromOutbound).receiverId);
		assertEquals(2, ((PartitionRequest) readFromOutbound).credit);

		readFromOutbound = channel.readOutbound();
		assertThat(readFromOutbound, instanceOf(PartitionRequest.class));
		assertEquals(inputChannel2.getInputChannelId(), ((PartitionRequest) readFromOutbound).receiverId);
		assertEquals(2, ((PartitionRequest) readFromOutbound).credit);

		// The buffer response will take one available buffer from input channel, and it will trigger
		// requesting (backlog + numExclusiveBuffers - numAvailableBuffers) floating buffers
		final BufferResponse bufferResponse1 = createBufferResponse(
			TestBufferFactory.createBuffer(32), 0, inputChannel1.getInputChannelId(), 1);
		final BufferResponse bufferResponse2 = createBufferResponse(
			TestBufferFactory.createBuffer(32), 0, inputChannel2.getInputChannelId(), 1);
		handler.channelRead(mock(ChannelHandlerContext.class), bufferResponse1);
		handler.channelRead(mock(ChannelHandlerContext.class), bufferResponse2);

		assertEquals(2, inputChannel1.getUnannouncedCredit());
		assertEquals(2, inputChannel2.getUnannouncedCredit());

		channel.runPendingTasks();

		// The two input channels should notify credits availability via the writable channel
		readFromOutbound = channel.readOutbound();
		assertThat(readFromOutbound, instanceOf(AddCredit.class));
		assertEquals(inputChannel1.getInputChannelId(), ((AddCredit) readFromOutbound).receiverId);
		assertEquals(2, ((AddCredit) readFromOutbound).credit);

		readFromOutbound = channel.readOutbound();
		assertThat(readFromOutbound, instanceOf(AddCredit.class));
		assertEquals(inputChannel2.getInputChannelId(), ((AddCredit) readFromOutbound).receiverId);
		assertEquals(2, ((AddCredit) readFromOutbound).credit);
		assertNull(channel.readOutbound());

		ByteBuf channelBlockingBuffer = blockChannel(channel);

		// Trigger notify credits availability via buffer response on the condition of an un-writable channel
		final BufferResponse bufferResponse3 = createBufferResponse(
			TestBufferFactory.createBuffer(32), 1, inputChannel1.getInputChannelId(), 1);
		handler.channelRead(mock(ChannelHandlerContext.class), bufferResponse3);

		assertEquals(1, inputChannel1.getUnannouncedCredit());
		assertEquals(0, inputChannel2.getUnannouncedCredit());

		channel.runPendingTasks();

		// The input channel will not notify credits via un-writable channel
		assertFalse(channel.isWritable());
		assertNull(channel.readOutbound());

		// Flush the buffer to make the channel writable again
		channel.flush();
		assertSame(channelBlockingBuffer, channel.readOutbound());

		// The input channel should notify credits via channel's writability changed event
		assertTrue(channel.isWritable());
		readFromOutbound = channel.readOutbound();
		assertThat(readFromOutbound, instanceOf(AddCredit.class));
		assertEquals(1, ((AddCredit) readFromOutbound).credit);
		assertEquals(0, inputChannel1.getUnannouncedCredit());
		assertEquals(0, inputChannel2.getUnannouncedCredit());

		// no more messages
		assertNull(channel.readOutbound());
	} finally {
		// Release all the buffer resources
		inputGate.close();

		networkBufferPool.destroyAllBufferPools();
		networkBufferPool.destroy();
	}
}
 
Example #24
Source File: CreditBasedPartitionRequestClientHandlerTest.java    From flink with Apache License 2.0 4 votes vote down vote up
private void testReadBufferResponseWithReleasingOrRemovingChannel(
	boolean isRemoved,
	boolean readBeforeReleasingOrRemoving) throws Exception {

	int bufferSize = 1024;

	NetworkBufferPool networkBufferPool = new NetworkBufferPool(10, bufferSize, 2);
	SingleInputGate inputGate = createSingleInputGate(1, networkBufferPool);
	RemoteInputChannel inputChannel = new InputChannelBuilder()
		.buildRemoteChannel(inputGate);
	inputGate.setInputChannels(inputChannel);
	inputGate.assignExclusiveSegments();

	CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
	EmbeddedChannel embeddedChannel = new EmbeddedChannel(handler);
	handler.addInputChannel(inputChannel);

	try {
		if (!readBeforeReleasingOrRemoving) {
			// Release the channel.
			inputGate.close();
			if (isRemoved) {
				handler.removeInputChannel(inputChannel);
			}
		}

		BufferResponse bufferResponse = createBufferResponse(
			TestBufferFactory.createBuffer(bufferSize),
			0,
			inputChannel.getInputChannelId(),
			1,
			new NetworkBufferAllocator(handler));

		if (readBeforeReleasingOrRemoving) {
			// Release the channel.
			inputGate.close();
			if (isRemoved) {
				handler.removeInputChannel(inputChannel);
			}
		}

		handler.channelRead(null, bufferResponse);

		assertEquals(0, inputChannel.getNumberOfQueuedBuffers());
		if (!readBeforeReleasingOrRemoving) {
			assertNull(bufferResponse.getBuffer());
		} else {
			assertNotNull(bufferResponse.getBuffer());
			assertTrue(bufferResponse.getBuffer().isRecycled());
		}

		embeddedChannel.runScheduledPendingTasks();
		NettyMessage.CancelPartitionRequest cancelPartitionRequest = embeddedChannel.readOutbound();
		assertNotNull(cancelPartitionRequest);
		assertEquals(inputChannel.getInputChannelId(), cancelPartitionRequest.receiverId);
	} finally {
		releaseResource(inputGate, networkBufferPool);
		embeddedChannel.close();
	}
}
 
Example #25
Source File: CreditBasedPartitionRequestClientHandlerTest.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Verifies that {@link RemoteInputChannel} is enqueued in the pipeline, but {@link AddCredit}
 * message is not sent actually when this input channel is released.
 */
@Test
public void testNotifyCreditAvailableAfterReleased() throws Exception {
	final CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
	final EmbeddedChannel channel = new EmbeddedChannel(handler);
	final PartitionRequestClient client = new NettyPartitionRequestClient(
		channel, handler, mock(ConnectionID.class), mock(PartitionRequestClientFactory.class));

	final NetworkBufferPool networkBufferPool = new NetworkBufferPool(10, 32, 2);
	final SingleInputGate inputGate = createSingleInputGate(1, networkBufferPool);
	final RemoteInputChannel inputChannel = createRemoteInputChannel(inputGate, client);
	try {
		inputGate.setInputChannels(inputChannel);
		final BufferPool bufferPool = networkBufferPool.createBufferPool(6, 6);
		inputGate.setBufferPool(bufferPool);
		inputGate.assignExclusiveSegments();

		inputChannel.requestSubpartition(0);

		// This should send the partition request
		Object readFromOutbound = channel.readOutbound();
		assertThat(readFromOutbound, instanceOf(PartitionRequest.class));
		assertEquals(2, ((PartitionRequest) readFromOutbound).credit);

		// Trigger request floating buffers via buffer response to notify credits available
		final BufferResponse bufferResponse = createBufferResponse(
			TestBufferFactory.createBuffer(32),
			0,
			inputChannel.getInputChannelId(),
			1,
			new NetworkBufferAllocator(handler));
		handler.channelRead(mock(ChannelHandlerContext.class), bufferResponse);

		assertEquals(2, inputChannel.getUnannouncedCredit());

		// Release the input channel
		inputGate.close();

		// it should send a close request after releasing the input channel,
		// but will not notify credits for a released input channel.
		readFromOutbound = channel.readOutbound();
		assertThat(readFromOutbound, instanceOf(CloseRequest.class));

		channel.runPendingTasks();

		assertNull(channel.readOutbound());
	} finally {
		releaseResource(inputGate, networkBufferPool);
		channel.close();
	}
}
 
Example #26
Source File: CreditBasedPartitionRequestClientHandlerTest.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Verifies that {@link BufferResponse} of compressed {@link Buffer} can be handled correctly.
 */
@Test
public void testReceiveCompressedBuffer() throws Exception {
	int bufferSize = 1024;
	String compressionCodec = "LZ4";
	BufferCompressor compressor = new BufferCompressor(bufferSize, compressionCodec);
	BufferDecompressor decompressor = new BufferDecompressor(bufferSize, compressionCodec);
	NetworkBufferPool networkBufferPool = new NetworkBufferPool(10, bufferSize, 2);
	SingleInputGate inputGate = new SingleInputGateBuilder()
		.setBufferDecompressor(decompressor)
		.setSegmentProvider(networkBufferPool)
		.build();
	RemoteInputChannel inputChannel = createRemoteInputChannel(inputGate, null);
	inputGate.setInputChannels(inputChannel);

	try {
		BufferPool bufferPool = networkBufferPool.createBufferPool(8, 8);
		inputGate.setBufferPool(bufferPool);
		inputGate.assignExclusiveSegments();

		CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
		handler.addInputChannel(inputChannel);

		Buffer buffer = compressor.compressToOriginalBuffer(TestBufferFactory.createBuffer(bufferSize));
		BufferResponse bufferResponse = createBufferResponse(
			buffer,
			0,
			inputChannel.getInputChannelId(),
			2,
			new NetworkBufferAllocator(handler));
		assertTrue(bufferResponse.isCompressed);
		handler.channelRead(null, bufferResponse);

		Buffer receivedBuffer = inputChannel.getNextReceivedBuffer();
		assertNotNull(receivedBuffer);
		assertTrue(receivedBuffer.isCompressed());
		receivedBuffer.recycleBuffer();
	} finally {
		releaseResource(inputGate, networkBufferPool);
	}
}
 
Example #27
Source File: CreditBasedPartitionRequestClientHandlerTest.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Verifies that {@link RemoteInputChannel} is enqueued in the pipeline, but {@link AddCredit}
 * message is not sent actually when this input channel is released.
 */
@Test
public void testNotifyCreditAvailableAfterReleased() throws Exception {
	final CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
	final EmbeddedChannel channel = new EmbeddedChannel(handler);
	final PartitionRequestClient client = new NettyPartitionRequestClient(
		channel, handler, mock(ConnectionID.class), mock(PartitionRequestClientFactory.class));

	final NetworkBufferPool networkBufferPool = new NetworkBufferPool(10, 32, 2);
	final SingleInputGate inputGate = createSingleInputGate(1);
	final RemoteInputChannel inputChannel = createRemoteInputChannel(inputGate, client, networkBufferPool);
	try {
		final BufferPool bufferPool = networkBufferPool.createBufferPool(6, 6);
		inputGate.setBufferPool(bufferPool);
		inputGate.assignExclusiveSegments();

		inputChannel.requestSubpartition(0);

		// This should send the partition request
		Object readFromOutbound = channel.readOutbound();
		assertThat(readFromOutbound, instanceOf(PartitionRequest.class));
		assertEquals(2, ((PartitionRequest) readFromOutbound).credit);

		// Trigger request floating buffers via buffer response to notify credits available
		final BufferResponse bufferResponse = createBufferResponse(
			TestBufferFactory.createBuffer(32), 0, inputChannel.getInputChannelId(), 1);
		handler.channelRead(mock(ChannelHandlerContext.class), bufferResponse);

		assertEquals(2, inputChannel.getUnannouncedCredit());

		// Release the input channel
		inputGate.close();

		// it should send a close request after releasing the input channel,
		// but will not notify credits for a released input channel.
		readFromOutbound = channel.readOutbound();
		assertThat(readFromOutbound, instanceOf(CloseRequest.class));

		channel.runPendingTasks();

		assertNull(channel.readOutbound());
	} finally {
		// Release all the buffer resources
		inputGate.close();

		networkBufferPool.destroyAllBufferPools();
		networkBufferPool.destroy();
	}
}
 
Example #28
Source File: CreditBasedPartitionRequestClientHandlerTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
/**
 * Verifies that {@link RemoteInputChannel} is enqueued in the pipeline for notifying credits,
 * and verifies the behaviour of credit notification by triggering channel's writability changed.
 */
@Test
public void testNotifyCreditAvailable() throws Exception {
	final CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
	final EmbeddedChannel channel = new EmbeddedChannel(handler);
	final PartitionRequestClient client = new PartitionRequestClient(
		channel, handler, mock(ConnectionID.class), mock(PartitionRequestClientFactory.class));

	final NetworkBufferPool networkBufferPool = new NetworkBufferPool(10, 32);
	final SingleInputGate inputGate = createSingleInputGate();
	final RemoteInputChannel inputChannel1 = createRemoteInputChannel(inputGate, client);
	final RemoteInputChannel inputChannel2 = createRemoteInputChannel(inputGate, client);
	try {
		final BufferPool bufferPool = networkBufferPool.createBufferPool(6, 6);
		inputGate.setBufferPool(bufferPool);
		final int numExclusiveBuffers = 2;
		inputGate.assignExclusiveSegments(networkBufferPool, numExclusiveBuffers);

		inputChannel1.requestSubpartition(0);
		inputChannel2.requestSubpartition(0);

		// The two input channels should send partition requests
		assertTrue(channel.isWritable());
		Object readFromOutbound = channel.readOutbound();
		assertThat(readFromOutbound, instanceOf(PartitionRequest.class));
		assertEquals(inputChannel1.getInputChannelId(), ((PartitionRequest) readFromOutbound).receiverId);
		assertEquals(2, ((PartitionRequest) readFromOutbound).credit);

		readFromOutbound = channel.readOutbound();
		assertThat(readFromOutbound, instanceOf(PartitionRequest.class));
		assertEquals(inputChannel2.getInputChannelId(), ((PartitionRequest) readFromOutbound).receiverId);
		assertEquals(2, ((PartitionRequest) readFromOutbound).credit);

		// The buffer response will take one available buffer from input channel, and it will trigger
		// requesting (backlog + numExclusiveBuffers - numAvailableBuffers) floating buffers
		final BufferResponse bufferResponse1 = createBufferResponse(
			TestBufferFactory.createBuffer(32), 0, inputChannel1.getInputChannelId(), 1);
		final BufferResponse bufferResponse2 = createBufferResponse(
			TestBufferFactory.createBuffer(32), 0, inputChannel2.getInputChannelId(), 1);
		handler.channelRead(mock(ChannelHandlerContext.class), bufferResponse1);
		handler.channelRead(mock(ChannelHandlerContext.class), bufferResponse2);

		assertEquals(2, inputChannel1.getUnannouncedCredit());
		assertEquals(2, inputChannel2.getUnannouncedCredit());

		channel.runPendingTasks();

		// The two input channels should notify credits availability via the writable channel
		readFromOutbound = channel.readOutbound();
		assertThat(readFromOutbound, instanceOf(AddCredit.class));
		assertEquals(inputChannel1.getInputChannelId(), ((AddCredit) readFromOutbound).receiverId);
		assertEquals(2, ((AddCredit) readFromOutbound).credit);

		readFromOutbound = channel.readOutbound();
		assertThat(readFromOutbound, instanceOf(AddCredit.class));
		assertEquals(inputChannel2.getInputChannelId(), ((AddCredit) readFromOutbound).receiverId);
		assertEquals(2, ((AddCredit) readFromOutbound).credit);
		assertNull(channel.readOutbound());

		ByteBuf channelBlockingBuffer = blockChannel(channel);

		// Trigger notify credits availability via buffer response on the condition of an un-writable channel
		final BufferResponse bufferResponse3 = createBufferResponse(
			TestBufferFactory.createBuffer(32), 1, inputChannel1.getInputChannelId(), 1);
		handler.channelRead(mock(ChannelHandlerContext.class), bufferResponse3);

		assertEquals(1, inputChannel1.getUnannouncedCredit());
		assertEquals(0, inputChannel2.getUnannouncedCredit());

		channel.runPendingTasks();

		// The input channel will not notify credits via un-writable channel
		assertFalse(channel.isWritable());
		assertNull(channel.readOutbound());

		// Flush the buffer to make the channel writable again
		channel.flush();
		assertSame(channelBlockingBuffer, channel.readOutbound());

		// The input channel should notify credits via channel's writability changed event
		assertTrue(channel.isWritable());
		readFromOutbound = channel.readOutbound();
		assertThat(readFromOutbound, instanceOf(AddCredit.class));
		assertEquals(1, ((AddCredit) readFromOutbound).credit);
		assertEquals(0, inputChannel1.getUnannouncedCredit());
		assertEquals(0, inputChannel2.getUnannouncedCredit());

		// no more messages
		assertNull(channel.readOutbound());
	} finally {
		// Release all the buffer resources
		inputGate.releaseAllResources();

		networkBufferPool.destroyAllBufferPools();
		networkBufferPool.destroy();
	}
}
 
Example #29
Source File: CreditBasedPartitionRequestClientHandlerTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
/**
 * Verifies that {@link RemoteInputChannel} is enqueued in the pipeline, but {@link AddCredit}
 * message is not sent actually when this input channel is released.
 */
@Test
public void testNotifyCreditAvailableAfterReleased() throws Exception {
	final CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
	final EmbeddedChannel channel = new EmbeddedChannel(handler);
	final PartitionRequestClient client = new PartitionRequestClient(
		channel, handler, mock(ConnectionID.class), mock(PartitionRequestClientFactory.class));

	final NetworkBufferPool networkBufferPool = new NetworkBufferPool(10, 32);
	final SingleInputGate inputGate = createSingleInputGate();
	final RemoteInputChannel inputChannel = createRemoteInputChannel(inputGate, client);
	try {
		final BufferPool bufferPool = networkBufferPool.createBufferPool(6, 6);
		inputGate.setBufferPool(bufferPool);
		final int numExclusiveBuffers = 2;
		inputGate.assignExclusiveSegments(networkBufferPool, numExclusiveBuffers);

		inputChannel.requestSubpartition(0);

		// This should send the partition request
		Object readFromOutbound = channel.readOutbound();
		assertThat(readFromOutbound, instanceOf(PartitionRequest.class));
		assertEquals(2, ((PartitionRequest) readFromOutbound).credit);

		// Trigger request floating buffers via buffer response to notify credits available
		final BufferResponse bufferResponse = createBufferResponse(
			TestBufferFactory.createBuffer(32), 0, inputChannel.getInputChannelId(), 1);
		handler.channelRead(mock(ChannelHandlerContext.class), bufferResponse);

		assertEquals(2, inputChannel.getUnannouncedCredit());

		// Release the input channel
		inputGate.releaseAllResources();

		// it should send a close request after releasing the input channel,
		// but will not notify credits for a released input channel.
		readFromOutbound = channel.readOutbound();
		assertThat(readFromOutbound, instanceOf(CloseRequest.class));

		channel.runPendingTasks();

		assertNull(channel.readOutbound());
	} finally {
		// Release all the buffer resources
		inputGate.releaseAllResources();

		networkBufferPool.destroyAllBufferPools();
		networkBufferPool.destroy();
	}
}