org.apache.flink.runtime.io.network.netty.exception.LocalTransportException Java Examples

/**
 * Sends a task event backwards to an intermediate result partition producer.
 * <p>
 * Backwards task events flow between readers and writers and therefore
 * will only work when both are running at the same time, which is only
 * guaranteed to be the case when both the respective producer and
 * consumer task run pipelined.
 */
public void sendTaskEvent(ResultPartitionID partitionId, TaskEvent event, final RemoteInputChannel inputChannel) throws IOException {
	checkNotClosed();

	tcpChannel.writeAndFlush(new TaskEventRequest(event, partitionId, inputChannel.getInputChannelId()))
			.addListener(
					new ChannelFutureListener() {
						@Override
						public void operationComplete(ChannelFuture future) throws Exception {
							if (!future.isSuccess()) {
								SocketAddress remoteAddr = future.channel().remoteAddress();
								inputChannel.onError(new LocalTransportException(
									String.format("Sending the task event to '%s' failed.", remoteAddr),
									future.channel().localAddress(), future.cause()
								));
							}
						}
					});
}
 

@Override
public void operationComplete(ChannelFuture future) throws Exception {
	if (future.isSuccess()) {
		handInChannel(future.channel());
	}
	else if (future.cause() != null) {
		notifyOfError(new RemoteTransportException(
				"Connecting to remote task manager + '" + connectionId.getAddress() +
						"' has failed. This might indicate that the remote task " +
						"manager has been lost.",
				connectionId.getAddress(), future.cause()));
	}
	else {
		notifyOfError(new LocalTransportException(
			String.format(
				"Connecting to remote task manager '%s' has been cancelled.",
				connectionId.getAddress()),
			null));
	}
}
 

/**
 * Sends a task event backwards to an intermediate result partition producer.
 *
 * <p>Backwards task events flow between readers and writers and therefore
 * will only work when both are running at the same time, which is only
 * guaranteed to be the case when both the respective producer and
 * consumer task run pipelined.
 */
@Override
public void sendTaskEvent(ResultPartitionID partitionId, TaskEvent event, final RemoteInputChannel inputChannel) throws IOException {
	checkNotClosed();

	tcpChannel.writeAndFlush(new TaskEventRequest(event, partitionId, inputChannel.getInputChannelId()))
			.addListener(
					new ChannelFutureListener() {
						@Override
						public void operationComplete(ChannelFuture future) throws Exception {
							if (!future.isSuccess()) {
								SocketAddress remoteAddr = future.channel().remoteAddress();
								inputChannel.onError(new LocalTransportException(
									String.format("Sending the task event to '%s' failed.", remoteAddr),
									future.channel().localAddress(), future.cause()
								));
							}
						}
					});
}
 

@Override
public void operationComplete(ChannelFuture future) throws Exception {
	if (future.isSuccess()) {
		handInChannel(future.channel());
	}
	else if (future.cause() != null) {
		notifyOfError(new RemoteTransportException(
				"Connecting to remote task manager + '" + connectionId.getAddress() +
						"' has failed. This might indicate that the remote task " +
						"manager has been lost.",
				connectionId.getAddress(), future.cause()));
	}
	else {
		notifyOfError(new LocalTransportException(
			String.format(
				"Connecting to remote task manager '%s' has been cancelled.",
				connectionId.getAddress()),
			null));
	}
}
 

/**
 * Sends a task event backwards to an intermediate result partition producer.
 *
 * <p>Backwards task events flow between readers and writers and therefore
 * will only work when both are running at the same time, which is only
 * guaranteed to be the case when both the respective producer and
 * consumer task run pipelined.
 */
@Override
public void sendTaskEvent(ResultPartitionID partitionId, TaskEvent event, final RemoteInputChannel inputChannel) throws IOException {
	checkNotClosed();

	tcpChannel.writeAndFlush(new TaskEventRequest(event, partitionId, inputChannel.getInputChannelId()))
			.addListener(
					new ChannelFutureListener() {
						@Override
						public void operationComplete(ChannelFuture future) throws Exception {
							if (!future.isSuccess()) {
								SocketAddress remoteAddr = future.channel().remoteAddress();
								inputChannel.onError(new LocalTransportException(
									String.format("Sending the task event to '%s' failed.", remoteAddr),
									future.channel().localAddress(), future.cause()
								));
							}
						}
					});
}
 

@Override
public void operationComplete(ChannelFuture future) throws Exception {
	if (future.isSuccess()) {
		handInChannel(future.channel());
	}
	else if (future.cause() != null) {
		notifyOfError(new RemoteTransportException(
				"Connecting to remote task manager + '" + connectionId.getAddress() +
						"' has failed. This might indicate that the remote task " +
						"manager has been lost.",
				connectionId.getAddress(), future.cause()));
	}
	else {
		notifyOfError(new LocalTransportException(
			String.format(
				"Connecting to remote task manager '%s' has been cancelled.",
				connectionId.getAddress()),
			null));
	}
}
 

private void checkNotClosed() throws IOException {
	if (closeReferenceCounter.isDisposed()) {
		final SocketAddress localAddr = tcpChannel.localAddress();
		final SocketAddress remoteAddr = tcpChannel.remoteAddress();
		throw new LocalTransportException(String.format("Channel to '%s' closed.", remoteAddr), localAddr);
	}
}
 

private void checkNotClosed() throws IOException {
	if (closeReferenceCounter.isDisposed()) {
		final SocketAddress localAddr = tcpChannel.localAddress();
		final SocketAddress remoteAddr = tcpChannel.remoteAddress();
		throw new LocalTransportException(String.format("Channel to '%s' closed.", remoteAddr), localAddr);
	}
}
 

private void checkNotClosed() throws IOException {
	if (closeReferenceCounter.isDisposed()) {
		final SocketAddress localAddr = tcpChannel.localAddress();
		final SocketAddress remoteAddr = tcpChannel.remoteAddress();
		throw new LocalTransportException(String.format("Channel to '%s' closed.", remoteAddr), localAddr);
	}
}
 

/**
 * Verifies that failed client requests via {@link PartitionRequestClient} are correctly
 * attributed to the respective {@link RemoteInputChannel}.
 */
@Test
public void testExceptionOnWrite() throws Exception {

	NettyProtocol protocol = new NettyProtocol(
			mock(ResultPartitionProvider.class),
			mock(TaskEventDispatcher.class),
			true) {

		@Override
		public ChannelHandler[] getServerChannelHandlers() {
			return new ChannelHandler[0];
		}
	};

	// We need a real server and client in this test, because Netty's EmbeddedChannel is
	// not failing the ChannelPromise of failed writes.
	NettyServerAndClient serverAndClient = initServerAndClient(protocol, createConfig());

	Channel ch = connect(serverAndClient);

	NetworkClientHandler handler = getClientHandler(ch);

	// Last outbound handler throws Exception after 1st write
	ch.pipeline().addFirst(new ChannelOutboundHandlerAdapter() {
		int writeNum = 0;

		@Override
		public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise)
				throws Exception {

			if (writeNum >= 1) {
				throw new RuntimeException("Expected test exception.");
			}

			writeNum++;
			ctx.write(msg, promise);
		}
	});

	PartitionRequestClient requestClient = new PartitionRequestClient(
			ch, handler, mock(ConnectionID.class), mock(PartitionRequestClientFactory.class));

	// Create input channels
	RemoteInputChannel[] rich = new RemoteInputChannel[] {
			createRemoteInputChannel(), createRemoteInputChannel()};

	final CountDownLatch sync = new CountDownLatch(1);

	// Do this with explicit synchronization. Otherwise this is not robust against slow timings
	// of the callback (e.g. we cannot just verify that it was called once, because there is
	// a chance that we do this too early).
	doAnswer(new Answer<Void>() {
		@Override
		public Void answer(InvocationOnMock invocation) throws Throwable {
			sync.countDown();
			return null;
		}
	}).when(rich[1]).onError(isA(LocalTransportException.class));

	// First request is successful
	ChannelFuture f = requestClient.requestSubpartition(new ResultPartitionID(), 0, rich[0], 0);
	assertTrue(f.await().isSuccess());

	// Second request is *not* successful
	f = requestClient.requestSubpartition(new ResultPartitionID(), 0, rich[1], 0);
	assertFalse(f.await().isSuccess());

	// Only the second channel should be notified about the error
	verify(rich[0], times(0)).onError(any(LocalTransportException.class));

	// Wait for the notification
	if (!sync.await(TestingUtils.TESTING_DURATION().toMillis(), TimeUnit.MILLISECONDS)) {
		fail("Timed out after waiting for " + TestingUtils.TESTING_DURATION().toMillis() +
				" ms to be notified about the channel error.");
	}

	shutdown(serverAndClient);
}
 

/**
 * Verifies that failed client requests via {@link PartitionRequestClient} are correctly
 * attributed to the respective {@link RemoteInputChannel}.
 */
@Test
public void testExceptionOnWrite() throws Exception {

	NettyProtocol protocol = new NettyProtocol(
			mock(ResultPartitionProvider.class),
			mock(TaskEventDispatcher.class),
			true) {

		@Override
		public ChannelHandler[] getServerChannelHandlers() {
			return new ChannelHandler[0];
		}
	};

	// We need a real server and client in this test, because Netty's EmbeddedChannel is
	// not failing the ChannelPromise of failed writes.
	NettyServerAndClient serverAndClient = initServerAndClient(protocol, createConfig());

	Channel ch = connect(serverAndClient);

	NetworkClientHandler handler = getClientHandler(ch);

	// Last outbound handler throws Exception after 1st write
	ch.pipeline().addFirst(new ChannelOutboundHandlerAdapter() {
		int writeNum = 0;

		@Override
		public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise)
				throws Exception {

			if (writeNum >= 1) {
				throw new RuntimeException("Expected test exception.");
			}

			writeNum++;
			ctx.write(msg, promise);
		}
	});

	PartitionRequestClient requestClient = new NettyPartitionRequestClient(
			ch, handler, mock(ConnectionID.class), mock(PartitionRequestClientFactory.class));

	// Create input channels
	RemoteInputChannel[] rich = new RemoteInputChannel[] {
			createRemoteInputChannel(), createRemoteInputChannel()};

	final CountDownLatch sync = new CountDownLatch(1);

	// Do this with explicit synchronization. Otherwise this is not robust against slow timings
	// of the callback (e.g. we cannot just verify that it was called once, because there is
	// a chance that we do this too early).
	doAnswer(new Answer<Void>() {
		@Override
		public Void answer(InvocationOnMock invocation) throws Throwable {
			sync.countDown();
			return null;
		}
	}).when(rich[1]).onError(isA(LocalTransportException.class));

	// First request is successful
	requestClient.requestSubpartition(new ResultPartitionID(), 0, rich[0], 0);

	// Second request is *not* successful
	requestClient.requestSubpartition(new ResultPartitionID(), 0, rich[1], 0);

	// Wait for the notification and it could confirm all the request operations are done
	if (!sync.await(TestingUtils.TESTING_DURATION().toMillis(), TimeUnit.MILLISECONDS)) {
		fail("Timed out after waiting for " + TestingUtils.TESTING_DURATION().toMillis() +
				" ms to be notified about the channel error.");
	}

	// Only the second channel should be notified about the error
	verify(rich[0], times(0)).onError(any(LocalTransportException.class));

	shutdown(serverAndClient);
}
 

/**
 * Verifies that failed client requests via {@link PartitionRequestClient} are correctly
 * attributed to the respective {@link RemoteInputChannel}.
 */
@Test
public void testExceptionOnWrite() throws Exception {

	NettyProtocol protocol = new NettyProtocol(
			mock(ResultPartitionProvider.class),
			mock(TaskEventDispatcher.class)) {

		@Override
		public ChannelHandler[] getServerChannelHandlers() {
			return new ChannelHandler[0];
		}
	};

	// We need a real server and client in this test, because Netty's EmbeddedChannel is
	// not failing the ChannelPromise of failed writes.
	NettyServerAndClient serverAndClient = initServerAndClient(protocol, createConfig());

	Channel ch = connect(serverAndClient);

	NetworkClientHandler handler = getClientHandler(ch);

	// Last outbound handler throws Exception after 1st write
	ch.pipeline().addFirst(new ChannelOutboundHandlerAdapter() {
		int writeNum = 0;

		@Override
		public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise)
				throws Exception {

			if (writeNum >= 1) {
				throw new RuntimeException("Expected test exception.");
			}

			writeNum++;
			ctx.write(msg, promise);
		}
	});

	PartitionRequestClient requestClient = new NettyPartitionRequestClient(
			ch, handler, mock(ConnectionID.class), mock(PartitionRequestClientFactory.class));

	// Create input channels
	RemoteInputChannel[] rich = new RemoteInputChannel[] {
			createRemoteInputChannel(), createRemoteInputChannel()};

	final CountDownLatch sync = new CountDownLatch(1);

	// Do this with explicit synchronization. Otherwise this is not robust against slow timings
	// of the callback (e.g. we cannot just verify that it was called once, because there is
	// a chance that we do this too early).
	doAnswer(new Answer<Void>() {
		@Override
		public Void answer(InvocationOnMock invocation) throws Throwable {
			sync.countDown();
			return null;
		}
	}).when(rich[1]).onError(isA(LocalTransportException.class));

	// First request is successful
	requestClient.requestSubpartition(new ResultPartitionID(), 0, rich[0], 0);

	// Second request is *not* successful
	requestClient.requestSubpartition(new ResultPartitionID(), 0, rich[1], 0);

	// Wait for the notification and it could confirm all the request operations are done
	if (!sync.await(TestingUtils.TESTING_DURATION().toMillis(), TimeUnit.MILLISECONDS)) {
		fail("Timed out after waiting for " + TestingUtils.TESTING_DURATION().toMillis() +
				" ms to be notified about the channel error.");
	}

	// Only the second channel should be notified about the error
	verify(rich[0], times(0)).onError(any(LocalTransportException.class));

	shutdown(serverAndClient);
}