org.apache.flink.runtime.io.network.NetworkClientHandler Java Examples

private void handInChannel(Channel channel) {
	synchronized (connectLock) {
		try {
			NetworkClientHandler clientHandler = channel.pipeline().get(NetworkClientHandler.class);
			partitionRequestClient = new PartitionRequestClient(
				channel, clientHandler, connectionId, clientFactory);

			if (disposeRequestClient) {
				partitionRequestClient.disposeIfNotUsed();
			}

			connectLock.notifyAll();
		}
		catch (Throwable t) {
			notifyOfError(t);
		}
	}
}
 

private void handInChannel(Channel channel) {
	synchronized (connectLock) {
		try {
			NetworkClientHandler clientHandler = channel.pipeline().get(NetworkClientHandler.class);
			partitionRequestClient = new NettyPartitionRequestClient(
				channel, clientHandler, connectionId, clientFactory);

			if (disposeRequestClient) {
				partitionRequestClient.disposeIfNotUsed();
			}

			connectLock.notifyAll();
		}
		catch (Throwable t) {
			notifyOfError(t);
		}
	}
}
 

private void handInChannel(Channel channel) {
	synchronized (connectLock) {
		try {
			NetworkClientHandler clientHandler = channel.pipeline().get(NetworkClientHandler.class);
			partitionRequestClient = new NettyPartitionRequestClient(
				channel, clientHandler, connectionId, clientFactory);

			if (disposeRequestClient) {
				partitionRequestClient.disposeIfNotUsed();
			}

			connectLock.notifyAll();
		}
		catch (Throwable t) {
			notifyOfError(t);
		}
	}
}
 

NettyPartitionRequestClient(
		Channel tcpChannel,
		NetworkClientHandler clientHandler,
		ConnectionID connectionId,
		PartitionRequestClientFactory clientFactory) {

	this.tcpChannel = checkNotNull(tcpChannel);
	this.clientHandler = checkNotNull(clientHandler);
	this.connectionId = checkNotNull(connectionId);
	this.clientFactory = checkNotNull(clientFactory);
}
 

private RemoteInputChannel addInputChannel(NetworkClientHandler clientHandler)
	throws IOException {
	RemoteInputChannel rich = createRemoteInputChannel();
	clientHandler.addInputChannel(rich);

	return rich;
}
 

/**
 * Verifies that "Connection reset by peer" Exceptions are special-cased and are reported as
 * an instance of {@link RemoteTransportException}.
 */
@Test
public void testConnectionResetByPeer() throws Throwable {
	EmbeddedChannel ch = createEmbeddedChannel();

	NetworkClientHandler handler = getClientHandler(ch);

	RemoteInputChannel rich = addInputChannel(handler);

	final Throwable[] error = new Throwable[1];

	// Verify the Exception
	doAnswer(new Answer<Void>() {
		@Override
		public Void answer(InvocationOnMock invocation) throws Throwable {
			Throwable cause = (Throwable) invocation.getArguments()[0];

			try {
				assertEquals(RemoteTransportException.class, cause.getClass());
				assertNotEquals("Connection reset by peer", cause.getMessage());

				assertEquals(IOException.class, cause.getCause().getClass());
				assertEquals("Connection reset by peer", cause.getCause().getMessage());
			}
			catch (Throwable t) {
				error[0] = t;
			}

			return null;
		}
	}).when(rich).onError(any(Throwable.class));

	ch.pipeline().fireExceptionCaught(new IOException("Connection reset by peer"));

	assertNull(error[0]);
}
 

private NettyPartitionRequestClient createPartitionRequestClient(
		Channel tcpChannel,
		NetworkClientHandler clientHandler) throws Exception {
	int port  = NetUtils.getAvailablePort();
	ConnectionID connectionID = new ConnectionID(new InetSocketAddress("localhost", port), 0);
	NettyConfig config = new NettyConfig(InetAddress.getLocalHost(), port, 1024, 1, new Configuration());
	NettyClient nettyClient = new NettyClient(config);
	PartitionRequestClientFactory partitionRequestClientFactory = new PartitionRequestClientFactory(nettyClient);

	return new NettyPartitionRequestClient(tcpChannel, clientHandler, connectionID, partitionRequestClientFactory);
}
 

NettyPartitionRequestClient(
		Channel tcpChannel,
		NetworkClientHandler clientHandler,
		ConnectionID connectionId,
		PartitionRequestClientFactory clientFactory) {

	this.tcpChannel = checkNotNull(tcpChannel);
	this.clientHandler = checkNotNull(clientHandler);
	this.connectionId = checkNotNull(connectionId);
	this.clientFactory = checkNotNull(clientFactory);
}
 

/**
 * Verifies that "Connection reset by peer" Exceptions are special-cased and are reported as
 * an instance of {@link RemoteTransportException}.
 */
@Test
public void testConnectionResetByPeer() throws Throwable {
	EmbeddedChannel ch = createEmbeddedChannel();

	NetworkClientHandler handler = getClientHandler(ch);

	RemoteInputChannel rich = addInputChannel(handler);

	final Throwable[] error = new Throwable[1];

	// Verify the Exception
	doAnswer(new Answer<Void>() {
		@Override
		public Void answer(InvocationOnMock invocation) throws Throwable {
			Throwable cause = (Throwable) invocation.getArguments()[0];

			try {
				assertEquals(RemoteTransportException.class, cause.getClass());
				assertNotEquals("Connection reset by peer", cause.getMessage());

				assertEquals(IOException.class, cause.getCause().getClass());
				assertEquals("Connection reset by peer", cause.getCause().getMessage());
			}
			catch (Throwable t) {
				error[0] = t;
			}

			return null;
		}
	}).when(rich).onError(any(Throwable.class));

	ch.pipeline().fireExceptionCaught(new IOException("Connection reset by peer"));

	assertNull(error[0]);
}
 

private RemoteInputChannel addInputChannel(NetworkClientHandler clientHandler)
	throws IOException {
	RemoteInputChannel rich = createRemoteInputChannel();
	clientHandler.addInputChannel(rich);

	return rich;
}
 

/**
 * Verifies that "Connection reset by peer" Exceptions are special-cased and are reported as
 * an instance of {@link RemoteTransportException}.
 */
@Test
public void testConnectionResetByPeer() throws Throwable {
	EmbeddedChannel ch = createEmbeddedChannel();

	NetworkClientHandler handler = getClientHandler(ch);

	RemoteInputChannel rich = addInputChannel(handler);

	final Throwable[] error = new Throwable[1];

	// Verify the Exception
	doAnswer(new Answer<Void>() {
		@Override
		public Void answer(InvocationOnMock invocation) throws Throwable {
			Throwable cause = (Throwable) invocation.getArguments()[0];

			try {
				assertEquals(RemoteTransportException.class, cause.getClass());
				assertNotEquals("Connection reset by peer", cause.getMessage());

				assertEquals(IOException.class, cause.getCause().getClass());
				assertEquals("Connection reset by peer", cause.getCause().getMessage());
			}
			catch (Throwable t) {
				error[0] = t;
			}

			return null;
		}
	}).when(rich).onError(any(Throwable.class));

	ch.pipeline().fireExceptionCaught(new IOException("Connection reset by peer"));

	assertNull(error[0]);
}
 

private RemoteInputChannel addInputChannel(NetworkClientHandler clientHandler)
	throws IOException {
	RemoteInputChannel rich = createRemoteInputChannel();
	clientHandler.addInputChannel(rich);

	return rich;
}
 

PartitionRequestClient(
		Channel tcpChannel,
		NetworkClientHandler clientHandler,
		ConnectionID connectionId,
		PartitionRequestClientFactory clientFactory) {

	this.tcpChannel = checkNotNull(tcpChannel);
	this.clientHandler = checkNotNull(clientHandler);
	this.connectionId = checkNotNull(connectionId);
	this.clientFactory = checkNotNull(clientFactory);
}
 

/**
 * Verifies that failed client requests via {@link PartitionRequestClient} are correctly
 * attributed to the respective {@link RemoteInputChannel}.
 */
@Test
public void testExceptionOnWrite() throws Exception {

	NettyProtocol protocol = new NettyProtocol(
			mock(ResultPartitionProvider.class),
			mock(TaskEventDispatcher.class)) {

		@Override
		public ChannelHandler[] getServerChannelHandlers() {
			return new ChannelHandler[0];
		}
	};

	// We need a real server and client in this test, because Netty's EmbeddedChannel is
	// not failing the ChannelPromise of failed writes.
	NettyServerAndClient serverAndClient = initServerAndClient(protocol, createConfig());

	Channel ch = connect(serverAndClient);

	NetworkClientHandler handler = getClientHandler(ch);

	// Last outbound handler throws Exception after 1st write
	ch.pipeline().addFirst(new ChannelOutboundHandlerAdapter() {
		int writeNum = 0;

		@Override
		public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise)
				throws Exception {

			if (writeNum >= 1) {
				throw new RuntimeException("Expected test exception.");
			}

			writeNum++;
			ctx.write(msg, promise);
		}
	});

	PartitionRequestClient requestClient = new NettyPartitionRequestClient(
			ch, handler, mock(ConnectionID.class), mock(PartitionRequestClientFactory.class));

	// Create input channels
	RemoteInputChannel[] rich = new RemoteInputChannel[] {
			createRemoteInputChannel(), createRemoteInputChannel()};

	final CountDownLatch sync = new CountDownLatch(1);

	// Do this with explicit synchronization. Otherwise this is not robust against slow timings
	// of the callback (e.g. we cannot just verify that it was called once, because there is
	// a chance that we do this too early).
	doAnswer(new Answer<Void>() {
		@Override
		public Void answer(InvocationOnMock invocation) throws Throwable {
			sync.countDown();
			return null;
		}
	}).when(rich[1]).onError(isA(LocalTransportException.class));

	// First request is successful
	requestClient.requestSubpartition(new ResultPartitionID(), 0, rich[0], 0);

	// Second request is *not* successful
	requestClient.requestSubpartition(new ResultPartitionID(), 0, rich[1], 0);

	// Wait for the notification and it could confirm all the request operations are done
	if (!sync.await(TestingUtils.TESTING_DURATION().toMillis(), TimeUnit.MILLISECONDS)) {
		fail("Timed out after waiting for " + TestingUtils.TESTING_DURATION().toMillis() +
				" ms to be notified about the channel error.");
	}

	// Only the second channel should be notified about the error
	verify(rich[0], times(0)).onError(any(LocalTransportException.class));

	shutdown(serverAndClient);
}
 

/**
 * Verifies that the channel is closed if there is an error *during* error notification.
 */
@Test
public void testChannelClosedOnExceptionDuringErrorNotification() throws Exception {
	EmbeddedChannel ch = createEmbeddedChannel();

	NetworkClientHandler handler = getClientHandler(ch);

	RemoteInputChannel rich = addInputChannel(handler);

	doThrow(new RuntimeException("Expected test exception"))
			.when(rich).onError(any(Throwable.class));

	ch.pipeline().fireExceptionCaught(new Exception());

	assertFalse(ch.isActive());
}
 

NetworkBufferAllocator(NetworkClientHandler networkClientHandler) {
	this.networkClientHandler = checkNotNull(networkClientHandler);
}
 

private NetworkClientHandler getClientHandler(Channel ch) {
	return ch.pipeline().get(NetworkClientHandler.class);
}
 

NettyMessageClientDecoderDelegate(NetworkClientHandler networkClientHandler) {
this.bufferResponseDecoder = new BufferResponseDecoder(
	new NetworkBufferAllocator(
		checkNotNull(networkClientHandler)));
      this.nonBufferResponseDecoder = new NonBufferResponseDecoder();
  }
 

private NetworkClientHandler getClientHandler(Channel ch) {
	return ch.pipeline().get(NetworkClientHandler.class);
}
 

/**
 * Verifies that the channel is closed if there is an error *during* error notification.
 */
@Test
public void testChannelClosedOnExceptionDuringErrorNotification() throws Exception {
	EmbeddedChannel ch = createEmbeddedChannel();

	NetworkClientHandler handler = getClientHandler(ch);

	RemoteInputChannel rich = addInputChannel(handler);

	doThrow(new RuntimeException("Expected test exception"))
			.when(rich).onError(any(Throwable.class));

	ch.pipeline().fireExceptionCaught(new Exception());

	assertFalse(ch.isActive());
}
 

/**
 * Verifies that failed client requests via {@link PartitionRequestClient} are correctly
 * attributed to the respective {@link RemoteInputChannel}.
 */
@Test
public void testExceptionOnWrite() throws Exception {

	NettyProtocol protocol = new NettyProtocol(
			mock(ResultPartitionProvider.class),
			mock(TaskEventDispatcher.class),
			true) {

		@Override
		public ChannelHandler[] getServerChannelHandlers() {
			return new ChannelHandler[0];
		}
	};

	// We need a real server and client in this test, because Netty's EmbeddedChannel is
	// not failing the ChannelPromise of failed writes.
	NettyServerAndClient serverAndClient = initServerAndClient(protocol, createConfig());

	Channel ch = connect(serverAndClient);

	NetworkClientHandler handler = getClientHandler(ch);

	// Last outbound handler throws Exception after 1st write
	ch.pipeline().addFirst(new ChannelOutboundHandlerAdapter() {
		int writeNum = 0;

		@Override
		public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise)
				throws Exception {

			if (writeNum >= 1) {
				throw new RuntimeException("Expected test exception.");
			}

			writeNum++;
			ctx.write(msg, promise);
		}
	});

	PartitionRequestClient requestClient = new NettyPartitionRequestClient(
			ch, handler, mock(ConnectionID.class), mock(PartitionRequestClientFactory.class));

	// Create input channels
	RemoteInputChannel[] rich = new RemoteInputChannel[] {
			createRemoteInputChannel(), createRemoteInputChannel()};

	final CountDownLatch sync = new CountDownLatch(1);

	// Do this with explicit synchronization. Otherwise this is not robust against slow timings
	// of the callback (e.g. we cannot just verify that it was called once, because there is
	// a chance that we do this too early).
	doAnswer(new Answer<Void>() {
		@Override
		public Void answer(InvocationOnMock invocation) throws Throwable {
			sync.countDown();
			return null;
		}
	}).when(rich[1]).onError(isA(LocalTransportException.class));

	// First request is successful
	requestClient.requestSubpartition(new ResultPartitionID(), 0, rich[0], 0);

	// Second request is *not* successful
	requestClient.requestSubpartition(new ResultPartitionID(), 0, rich[1], 0);

	// Wait for the notification and it could confirm all the request operations are done
	if (!sync.await(TestingUtils.TESTING_DURATION().toMillis(), TimeUnit.MILLISECONDS)) {
		fail("Timed out after waiting for " + TestingUtils.TESTING_DURATION().toMillis() +
				" ms to be notified about the channel error.");
	}

	// Only the second channel should be notified about the error
	verify(rich[0], times(0)).onError(any(LocalTransportException.class));

	shutdown(serverAndClient);
}
 

private NetworkClientHandler getClientHandler(Channel ch) {
	return ch.pipeline().get(NetworkClientHandler.class);
}
 

/**
 * Verifies that the channel is closed if there is an error *during* error notification.
 */
@Test
public void testChannelClosedOnExceptionDuringErrorNotification() throws Exception {
	EmbeddedChannel ch = createEmbeddedChannel();

	NetworkClientHandler handler = getClientHandler(ch);

	RemoteInputChannel rich = addInputChannel(handler);

	doThrow(new RuntimeException("Expected test exception"))
			.when(rich).onError(any(Throwable.class));

	ch.pipeline().fireExceptionCaught(new Exception());

	assertFalse(ch.isActive());
}
 

/**
 * Verifies that failed client requests via {@link PartitionRequestClient} are correctly
 * attributed to the respective {@link RemoteInputChannel}.
 */
@Test
public void testExceptionOnWrite() throws Exception {

	NettyProtocol protocol = new NettyProtocol(
			mock(ResultPartitionProvider.class),
			mock(TaskEventDispatcher.class),
			true) {

		@Override
		public ChannelHandler[] getServerChannelHandlers() {
			return new ChannelHandler[0];
		}
	};

	// We need a real server and client in this test, because Netty's EmbeddedChannel is
	// not failing the ChannelPromise of failed writes.
	NettyServerAndClient serverAndClient = initServerAndClient(protocol, createConfig());

	Channel ch = connect(serverAndClient);

	NetworkClientHandler handler = getClientHandler(ch);

	// Last outbound handler throws Exception after 1st write
	ch.pipeline().addFirst(new ChannelOutboundHandlerAdapter() {
		int writeNum = 0;

		@Override
		public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise)
				throws Exception {

			if (writeNum >= 1) {
				throw new RuntimeException("Expected test exception.");
			}

			writeNum++;
			ctx.write(msg, promise);
		}
	});

	PartitionRequestClient requestClient = new PartitionRequestClient(
			ch, handler, mock(ConnectionID.class), mock(PartitionRequestClientFactory.class));

	// Create input channels
	RemoteInputChannel[] rich = new RemoteInputChannel[] {
			createRemoteInputChannel(), createRemoteInputChannel()};

	final CountDownLatch sync = new CountDownLatch(1);

	// Do this with explicit synchronization. Otherwise this is not robust against slow timings
	// of the callback (e.g. we cannot just verify that it was called once, because there is
	// a chance that we do this too early).
	doAnswer(new Answer<Void>() {
		@Override
		public Void answer(InvocationOnMock invocation) throws Throwable {
			sync.countDown();
			return null;
		}
	}).when(rich[1]).onError(isA(LocalTransportException.class));

	// First request is successful
	ChannelFuture f = requestClient.requestSubpartition(new ResultPartitionID(), 0, rich[0], 0);
	assertTrue(f.await().isSuccess());

	// Second request is *not* successful
	f = requestClient.requestSubpartition(new ResultPartitionID(), 0, rich[1], 0);
	assertFalse(f.await().isSuccess());

	// Only the second channel should be notified about the error
	verify(rich[0], times(0)).onError(any(LocalTransportException.class));

	// Wait for the notification
	if (!sync.await(TestingUtils.TESTING_DURATION().toMillis(), TimeUnit.MILLISECONDS)) {
		fail("Timed out after waiting for " + TestingUtils.TESTING_DURATION().toMillis() +
				" ms to be notified about the channel error.");
	}

	shutdown(serverAndClient);
}
 

/**
 * Returns the client channel handlers.
 *
 * <pre>
 *     +-----------+----------+            +----------------------+
 *     | Remote input channel |            | request client       |
 *     +-----------+----------+            +-----------+----------+
 *                 |                                   | (1) write
 * +---------------+-----------------------------------+---------------+
 * |               |     CLIENT CHANNEL PIPELINE       |               |
 * |               |                                  \|/              |
 * |    +----------+----------+            +----------------------+    |
 * |    | Request handler     +            | Message encoder      |    |
 * |    +----------+----------+            +-----------+----------+    |
 * |              /|\                                 \|/              |
 * |               |                                   |               |
 * |    +----------+------------+                      |               |
 * |    | Message+Frame decoder |                      |               |
 * |    +----------+------------+                      |               |
 * |              /|\                                  |               |
 * +---------------+-----------------------------------+---------------+
 * |               | (3) server response              \|/ (2) client request
 * +---------------+-----------------------------------+---------------+
 * |               |                                   |               |
 * |       [ Socket.read() ]                    [ Socket.write() ]     |
 * |                                                                   |
 * |  Netty Internal I/O Threads (Transport Implementation)            |
 * +-------------------------------------------------------------------+
 * </pre>
 *
 * @return channel handlers
 */
public ChannelHandler[] getClientChannelHandlers() {
	NetworkClientHandler networkClientHandler = new CreditBasedPartitionRequestClientHandler();

	return new ChannelHandler[]{
		messageEncoder,
		new NettyMessageClientDecoderDelegate(networkClientHandler),
		networkClientHandler};
}
 

/**
 * Returns the client channel handlers.
 *
 * <pre>
 *     +-----------+----------+            +----------------------+
 *     | Remote input channel |            | request client       |
 *     +-----------+----------+            +-----------+----------+
 *                 |                                   | (1) write
 * +---------------+-----------------------------------+---------------+
 * |               |     CLIENT CHANNEL PIPELINE       |               |
 * |               |                                  \|/              |
 * |    +----------+----------+            +----------------------+    |
 * |    | Request handler     +            | Message encoder      |    |
 * |    +----------+----------+            +-----------+----------+    |
 * |              /|\                                 \|/              |
 * |               |                                   |               |
 * |    +----------+------------+                      |               |
 * |    | Message+Frame decoder |                      |               |
 * |    +----------+------------+                      |               |
 * |              /|\                                  |               |
 * +---------------+-----------------------------------+---------------+
 * |               | (3) server response              \|/ (2) client request
 * +---------------+-----------------------------------+---------------+
 * |               |                                   |               |
 * |       [ Socket.read() ]                    [ Socket.write() ]     |
 * |                                                                   |
 * |  Netty Internal I/O Threads (Transport Implementation)            |
 * +-------------------------------------------------------------------+
 * </pre>
 *
 * @return channel handlers
 */
public ChannelHandler[] getClientChannelHandlers() {
	NetworkClientHandler networkClientHandler =
		creditBasedEnabled ? new CreditBasedPartitionRequestClientHandler() :
			new PartitionRequestClientHandler();
	return new ChannelHandler[] {
		messageEncoder,
		new NettyMessage.NettyMessageDecoder(!creditBasedEnabled),
		networkClientHandler};
}
 

/**
 * Returns the client channel handlers.
 *
 * <pre>
 *     +-----------+----------+            +----------------------+
 *     | Remote input channel |            | request client       |
 *     +-----------+----------+            +-----------+----------+
 *                 |                                   | (1) write
 * +---------------+-----------------------------------+---------------+
 * |               |     CLIENT CHANNEL PIPELINE       |               |
 * |               |                                  \|/              |
 * |    +----------+----------+            +----------------------+    |
 * |    | Request handler     +            | Message encoder      |    |
 * |    +----------+----------+            +-----------+----------+    |
 * |              /|\                                 \|/              |
 * |               |                                   |               |
 * |    +----------+------------+                      |               |
 * |    | Message+Frame decoder |                      |               |
 * |    +----------+------------+                      |               |
 * |              /|\                                  |               |
 * +---------------+-----------------------------------+---------------+
 * |               | (3) server response              \|/ (2) client request
 * +---------------+-----------------------------------+---------------+
 * |               |                                   |               |
 * |       [ Socket.read() ]                    [ Socket.write() ]     |
 * |                                                                   |
 * |  Netty Internal I/O Threads (Transport Implementation)            |
 * +-------------------------------------------------------------------+
 * </pre>
 *
 * @return channel handlers
 */
public ChannelHandler[] getClientChannelHandlers() {
	NetworkClientHandler networkClientHandler =
		creditBasedEnabled ? new CreditBasedPartitionRequestClientHandler() :
			new PartitionRequestClientHandler();
	return new ChannelHandler[] {
		messageEncoder,
		new NettyMessage.NettyMessageDecoder(!creditBasedEnabled),
		networkClientHandler};
}