org.apache.flink.shaded.netty4.io.netty.channel.Channel Java Examples

private Channel createServerChannel(final ChannelHandler... handlers) throws UnknownHostException, InterruptedException {
	ServerBootstrap bootstrap = new ServerBootstrap()
			// Bind address and port
			.localAddress(InetAddress.getLocalHost(), 0)
			// NIO server channels
			.group(nioGroup)
			.channel(NioServerSocketChannel.class)
			// See initializer for pipeline details
			.childHandler(new ChannelInitializer<SocketChannel>() {
				@Override
				protected void initChannel(SocketChannel ch) throws Exception {
					ch.pipeline()
							.addLast(new LengthFieldBasedFrameDecoder(Integer.MAX_VALUE, 0, 4, 0, 4))
							.addLast(handlers);
				}
			});

	return bootstrap.bind().sync().channel();
}
 

private Channel createServerChannel(final ChannelHandler... handlers) throws UnknownHostException, InterruptedException {
	ServerBootstrap bootstrap = new ServerBootstrap()
			// Bind address and port
			.localAddress(InetAddress.getLocalHost(), 0)
			// NIO server channels
			.group(nioGroup)
			.channel(NioServerSocketChannel.class)
			// See initializer for pipeline details
			.childHandler(new ChannelInitializer<SocketChannel>() {
				@Override
				protected void initChannel(SocketChannel ch) throws Exception {
					ch.pipeline()
							.addLast(new LengthFieldBasedFrameDecoder(Integer.MAX_VALUE, 0, 4, 0, 4))
							.addLast(handlers);
				}
			});

	return bootstrap.bind().sync().channel();
}
 

@Override
public void channelActive(ChannelHandlerContext ctx) throws Exception {
	final Channel ch = ctx.channel();

	assertEquals("Low watermark", expectedLowWatermark, ch.config().getWriteBufferLowWaterMark());
	assertEquals("High watermark", expectedHighWatermark, ch.config().getWriteBufferHighWaterMark());

	// Start with a writable channel
	assertTrue(ch.isWritable());

	// First buffer should not change writability
	ch.write(buffer());
	assertTrue(ch.isWritable());

	// ...second buffer should though
	ch.write(buffer());
	assertFalse(ch.isWritable());

	// Flush everything and close the channel after the writability changed event is fired.
	hasFlushed = true;
	ch.flush();
}
 

@Override
public void channelActive(ChannelHandlerContext ctx) throws Exception {
	final Channel ch = ctx.channel();

	assertEquals("Low watermark", expectedLowWatermark, ch.config().getWriteBufferLowWaterMark());
	assertEquals("High watermark", expectedHighWatermark, ch.config().getWriteBufferHighWaterMark());

	// Start with a writable channel
	assertTrue(ch.isWritable());

	// First buffer should not change writability
	ch.write(buffer());
	assertTrue(ch.isWritable());

	// ...second buffer should though
	ch.write(buffer());
	assertFalse(ch.isWritable());

	// Flush everything and close the channel after the writability changed event is fired.
	hasFlushed = true;
	ch.flush();
}
 

@Test
public void testClientUntrustedCertificate() throws Exception {
	final Configuration serverConfig = createSslConfig();
	final Configuration clientConfig = createSslConfig();

	// give the client a different keystore / certificate
	clientConfig.setString(SecurityOptions.SSL_INTERNAL_KEYSTORE, "src/test/resources/untrusted.keystore");

	final NettyConfig nettyServerConfig = createNettyConfig(serverConfig);
	final NettyConfig nettyClientConfig = createNettyConfig(clientConfig);

	final NettyBufferPool bufferPool = new NettyBufferPool(1);
	final NettyProtocol protocol = new NoOpProtocol();

	final NettyServer server = NettyTestUtil.initServer(nettyServerConfig, protocol, bufferPool);
	final NettyClient client = NettyTestUtil.initClient(nettyClientConfig, protocol, bufferPool);
	final NettyServerAndClient serverAndClient = new NettyServerAndClient(server, client);

	final Channel ch = NettyTestUtil.connect(serverAndClient);
	ch.pipeline().addLast(new StringDecoder()).addLast(new StringEncoder());

	// Attempting to write data over ssl should fail
	assertFalse(ch.writeAndFlush("test").await().isSuccess());

	NettyTestUtil.shutdown(serverAndClient);
}
 

private void handInChannel(Channel channel) {
	synchronized (connectLock) {
		try {
			NetworkClientHandler clientHandler = channel.pipeline().get(NetworkClientHandler.class);
			partitionRequestClient = new NettyPartitionRequestClient(
				channel, clientHandler, connectionId, clientFactory);

			if (disposeRequestClient) {
				partitionRequestClient.disposeIfNotUsed();
			}

			connectLock.notifyAll();
		}
		catch (Throwable t) {
			notifyOfError(t);
		}
	}
}
 

private Channel createServerChannel(final ChannelHandler... handlers) throws UnknownHostException, InterruptedException {
	ServerBootstrap bootstrap = new ServerBootstrap()
			// Bind address and port
			.localAddress(InetAddress.getLocalHost(), 0)
			// NIO server channels
			.group(nioGroup)
			.channel(NioServerSocketChannel.class)
			// See initializer for pipeline details
			.childHandler(new ChannelInitializer<SocketChannel>() {
				@Override
				protected void initChannel(SocketChannel ch) throws Exception {
					ch.pipeline()
							.addLast(new LengthFieldBasedFrameDecoder(Integer.MAX_VALUE, 0, 4, 0, 4))
							.addLast(handlers);
				}
			});

	return bootstrap.bind().sync().channel();
}
 

@Test
public void testClientUntrustedCertificate() throws Exception {
	final Configuration serverConfig = createSslConfig();
	final Configuration clientConfig = createSslConfig();

	// give the client a different keystore / certificate
	clientConfig.setString(SecurityOptions.SSL_INTERNAL_KEYSTORE, "src/test/resources/untrusted.keystore");

	final NettyConfig nettyServerConfig = createNettyConfig(serverConfig);
	final NettyConfig nettyClientConfig = createNettyConfig(clientConfig);

	final NettyBufferPool bufferPool = new NettyBufferPool(1);
	final NettyProtocol protocol = new NoOpProtocol();

	final NettyServer server = NettyTestUtil.initServer(nettyServerConfig, protocol, bufferPool);
	final NettyClient client = NettyTestUtil.initClient(nettyClientConfig, protocol, bufferPool);
	final NettyServerAndClient serverAndClient = new NettyServerAndClient(server, client);

	final Channel ch = NettyTestUtil.connect(serverAndClient);
	ch.pipeline().addLast(new StringDecoder()).addLast(new StringEncoder());

	// Attempting to write data over ssl should fail
	assertFalse(ch.writeAndFlush("test").await().isSuccess());

	NettyTestUtil.shutdown(serverAndClient);
}
 

/**
 * Creates an established connection with the given channel.
 *
 * @param serverAddress Address of the server connected to
 * @param channel The actual TCP channel
 */
EstablishedConnection(
		final InetSocketAddress serverAddress,
		final MessageSerializer<REQ, RESP> serializer,
		final Channel channel) {

	this.serverAddress = Preconditions.checkNotNull(serverAddress);
	this.channel = Preconditions.checkNotNull(channel);

	// Add the client handler with the callback
	channel.pipeline().addLast(
			getClientName() + " Handler",
			new ClientHandler<>(clientName, serializer, this)
	);

	stats.reportActiveConnection();
}
 

private void handInChannel(Channel channel) {
	synchronized (connectLock) {
		try {
			NetworkClientHandler clientHandler = channel.pipeline().get(NetworkClientHandler.class);
			partitionRequestClient = new PartitionRequestClient(
				channel, clientHandler, connectionId, clientFactory);

			if (disposeRequestClient) {
				partitionRequestClient.disposeIfNotUsed();
			}

			connectLock.notifyAll();
		}
		catch (Throwable t) {
			notifyOfError(t);
		}
	}
}
 

/**
 * Sends a request to to the server.
 *
 * <pre>
 * HttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/overview");
 * request.headers().set(HttpHeaders.Names.HOST, host);
 * request.headers().set(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.CLOSE);
 *
 * sendRequest(request);
 * </pre>
 *
 * @param request The {@link HttpRequest} to send to the server
 */
public void sendRequest(HttpRequest request, Duration timeout) throws InterruptedException, TimeoutException {
	LOG.debug("Writing {}.", request);

	// Make the connection attempt.
	ChannelFuture connect = bootstrap.connect(host, port);

	Channel channel;
	if (connect.await(timeout.toMillis(), TimeUnit.MILLISECONDS)) {
		channel = connect.channel();
	}
	else {
		throw new TimeoutException("Connection failed");
	}

	channel.writeAndFlush(request);
}
 

/**
 * Returns the address on which this endpoint is accepting requests.
 *
 * @return address on which this endpoint is accepting requests or null if none
 */
@Nullable
public InetSocketAddress getServerAddress() {
	synchronized (lock) {
		Preconditions.checkState(state != State.CREATED, "The RestServerEndpoint has not been started yet.");
		Channel server = this.serverChannel;

		if (server != null) {
			try {
				return ((InetSocketAddress) server.localAddress());
			} catch (Exception e) {
				log.error("Cannot access local server address", e);
			}
		}

		return null;
	}
}
 

/**
 * Creates an established connection with the given channel.
 *
 * @param serverAddress Address of the server connected to
 * @param channel The actual TCP channel
 */
EstablishedConnection(
		final InetSocketAddress serverAddress,
		final MessageSerializer<REQ, RESP> serializer,
		final Channel channel) {

	this.serverAddress = Preconditions.checkNotNull(serverAddress);
	this.channel = Preconditions.checkNotNull(channel);

	// Add the client handler with the callback
	channel.pipeline().addLast(
			getClientName() + " Handler",
			new ClientHandler<>(clientName, serializer, this)
	);

	stats.reportActiveConnection();
}
 

/**
 * Returns the address on which this endpoint is accepting requests.
 *
 * @return address on which this endpoint is accepting requests or null if none
 */
@Nullable
public InetSocketAddress getServerAddress() {
	synchronized (lock) {
		Preconditions.checkState(state != State.CREATED, "The RestServerEndpoint has not been started yet.");
		Channel server = this.serverChannel;

		if (server != null) {
			try {
				return ((InetSocketAddress) server.localAddress());
			} catch (Exception e) {
				log.error("Cannot access local server address", e);
			}
		}

		return null;
	}
}
 

private void handInChannel(Channel channel) {
	synchronized (connectLock) {
		try {
			NetworkClientHandler clientHandler = channel.pipeline().get(NetworkClientHandler.class);
			partitionRequestClient = new NettyPartitionRequestClient(
				channel, clientHandler, connectionId, clientFactory);

			if (disposeRequestClient) {
				partitionRequestClient.disposeIfNotUsed();
			}

			connectLock.notifyAll();
		}
		catch (Throwable t) {
			notifyOfError(t);
		}
	}
}
 

/**
 * Returns the address on which this endpoint is accepting requests.
 *
 * @return address on which this endpoint is accepting requests or null if none
 */
@Nullable
public InetSocketAddress getServerAddress() {
	synchronized (lock) {
		Preconditions.checkState(state != State.CREATED, "The RestServerEndpoint has not been started yet.");
		Channel server = this.serverChannel;

		if (server != null) {
			try {
				return ((InetSocketAddress) server.localAddress());
			} catch (Exception e) {
				log.error("Cannot access local server address", e);
			}
		}

		return null;
	}
}
 

/**
 * Sends a request to to the server.
 *
 * <pre>
 * HttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/overview");
 * request.headers().set(HttpHeaders.Names.HOST, host);
 * request.headers().set(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.CLOSE);
 *
 * sendRequest(request);
 * </pre>
 *
 * @param request The {@link HttpRequest} to send to the server
 */
public void sendRequest(HttpRequest request, FiniteDuration timeout) throws InterruptedException, TimeoutException {
	LOG.debug("Writing {}.", request);

	// Make the connection attempt.
	ChannelFuture connect = bootstrap.connect(host, port);

	Channel channel;
	if (connect.await(timeout.toMillis(), TimeUnit.MILLISECONDS)) {
		channel = connect.channel();
	}
	else {
		throw new TimeoutException("Connection failed");
	}

	channel.writeAndFlush(request);
}
 

/**
 * Sends a request to to the server.
 *
 * <pre>
 * HttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/overview");
 * request.headers().set(HttpHeaders.Names.HOST, host);
 * request.headers().set(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.CLOSE);
 *
 * sendRequest(request);
 * </pre>
 *
 * @param request The {@link HttpRequest} to send to the server
 */
public void sendRequest(HttpRequest request, FiniteDuration timeout) throws InterruptedException, TimeoutException {
	LOG.debug("Writing {}.", request);

	// Make the connection attempt.
	ChannelFuture connect = bootstrap.connect(host, port);

	Channel channel;
	if (connect.await(timeout.toMillis(), TimeUnit.MILLISECONDS)) {
		channel = connect.channel();
	}
	else {
		throw new TimeoutException("Connection failed");
	}

	channel.writeAndFlush(request);
}
 

@Test
public void testFileCleanup() throws Exception {
	final Path dir = temporaryFolder.newFolder().toPath();
	final Path file = dir.resolve("file");
	Files.createFile(file);

	RestfulGateway mockRestfulGateway = new TestingRestfulGateway.Builder()
		.build();

	final GatewayRetriever<RestfulGateway> mockGatewayRetriever = () ->
		CompletableFuture.completedFuture(mockRestfulGateway);

	CompletableFuture<Void> requestProcessingCompleteFuture = new CompletableFuture<>();
	TestHandler handler = new TestHandler(requestProcessingCompleteFuture, mockGatewayRetriever);

	RouteResult<?> routeResult = new RouteResult<>("", "", Collections.emptyMap(), Collections.emptyMap(), "");
	HttpRequest request = new DefaultFullHttpRequest(
		HttpVersion.HTTP_1_1,
		HttpMethod.GET,
		TestHandler.TestHeaders.INSTANCE.getTargetRestEndpointURL(),
		Unpooled.wrappedBuffer(new byte[0]));
	RoutedRequest<?> routerRequest = new RoutedRequest<>(routeResult, request);

	Attribute<FileUploads> attribute = new SimpleAttribute();
	attribute.set(new FileUploads(dir));
	Channel channel = mock(Channel.class);
	when(channel.attr(any(AttributeKey.class))).thenReturn(attribute);

	ChannelHandlerContext context = mock(ChannelHandlerContext.class);
	when(context.channel()).thenReturn(channel);

	handler.respondAsLeader(context, routerRequest, mockRestfulGateway);

	// the (asynchronous) request processing is not yet complete so the files should still exist
	Assert.assertTrue(Files.exists(file));
	requestProcessingCompleteFuture.complete(null);
	Assert.assertFalse(Files.exists(file));
}
 

private void handleException(Channel channel, Throwable cause) throws IOException {
	LOG.error("Encountered error while consuming partitions", cause);

	fatalError = true;
	releaseAllResources();

	if (channel.isActive()) {
		channel.writeAndFlush(new ErrorResponse(cause)).addListener(ChannelFutureListener.CLOSE);
	}
}
 

/**
 * Verifies that {@link RemoteInputChannel#onFailedPartitionRequest()} is called when a
 * {@link PartitionNotFoundException} is received.
 */
@Test
public void testReceivePartitionNotFoundException() throws Exception {
	// Minimal mock of a remote input channel
	final BufferProvider bufferProvider = mock(BufferProvider.class);
	when(bufferProvider.requestBuffer()).thenReturn(TestBufferFactory.createBuffer(0));

	final RemoteInputChannel inputChannel = mock(RemoteInputChannel.class);
	when(inputChannel.getInputChannelId()).thenReturn(new InputChannelID());
	when(inputChannel.getBufferProvider()).thenReturn(bufferProvider);

	final ErrorResponse partitionNotFound = new ErrorResponse(
		new PartitionNotFoundException(new ResultPartitionID()),
		inputChannel.getInputChannelId());

	final CreditBasedPartitionRequestClientHandler client = new CreditBasedPartitionRequestClientHandler();
	client.addInputChannel(inputChannel);

	// Mock channel context
	ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);
	when(ctx.channel()).thenReturn(mock(Channel.class));

	client.channelActive(ctx);

	client.channelRead(ctx, partitionNotFound);

	verify(inputChannel, times(1)).onFailedPartitionRequest();
}
 

/**
 * Get the server port on which the artifact server is listening.
 */
public synchronized int getServerPort() {
	Channel server = this.serverChannel;
	if (server != null) {
		try {
			return ((InetSocketAddress) server.localAddress()).getPort();
		} catch (Exception e) {
			LOG.error("Cannot access local server port", e);
		}
	}
	return -1;
}
 

NettyPartitionRequestClient(
		Channel tcpChannel,
		NetworkClientHandler clientHandler,
		ConnectionID connectionId,
		PartitionRequestClientFactory clientFactory) {

	this.tcpChannel = checkNotNull(tcpChannel);
	this.clientHandler = checkNotNull(clientHandler);
	this.connectionId = checkNotNull(connectionId);
	this.clientFactory = checkNotNull(clientFactory);
}
 

@Override
public void writeTo(Channel channel) {
	ChannelFuture future = channel.writeAndFlush(httpRequest);
	// this should never be false as we explicitly set the encoder to use multipart messages
	if (bodyRequestEncoder.isChunked()) {
		future = channel.writeAndFlush(bodyRequestEncoder);
	}

	// release data and remove temporary files if they were created, once the writing is complete
	future.addListener((ignored) -> bodyRequestEncoder.cleanFiles());
}
 

/**
 * Tries to write&flush unannounced credits for the next input channel in queue.
 *
 * <p>This method may be called by the first input channel enqueuing, or the complete
 * future's callback in previous input channel, or the channel writability changed event.
 */
private void writeAndFlushNextMessageIfPossible(Channel channel) {
	if (channelError.get() != null || !channel.isWritable()) {
		return;
	}

	while (true) {
		ClientOutboundMessage outboundMessage = clientOutboundMessages.poll();

		// The input channel may be null because of the write callbacks
		// that are executed after each write.
		if (outboundMessage == null) {
			return;
		}

		//It is no need to notify credit or resume data consumption for the released channel.
		if (!outboundMessage.inputChannel.isReleased()) {
			Object msg = outboundMessage.buildMessage();

			// Write and flush and wait until this is done before
			// trying to continue with the next input channel.
			channel.writeAndFlush(msg).addListener(writeListener);

			return;
		}
	}
}
 

@Test
public void testFileCleanup() throws Exception {
	final Path dir = temporaryFolder.newFolder().toPath();
	final Path file = dir.resolve("file");
	Files.createFile(file);

	RestfulGateway mockRestfulGateway = TestingRestfulGateway.newBuilder()
		.build();

	final GatewayRetriever<RestfulGateway> mockGatewayRetriever = () ->
		CompletableFuture.completedFuture(mockRestfulGateway);

	CompletableFuture<Void> requestProcessingCompleteFuture = new CompletableFuture<>();
	TestHandler handler = new TestHandler(requestProcessingCompleteFuture, mockGatewayRetriever);

	RouteResult<?> routeResult = new RouteResult<>("", "", Collections.emptyMap(), Collections.emptyMap(), "");
	HttpRequest request = new DefaultFullHttpRequest(
		HttpVersion.HTTP_1_1,
		HttpMethod.GET,
		TestHandler.TestHeaders.INSTANCE.getTargetRestEndpointURL(),
		Unpooled.wrappedBuffer(new byte[0]));
	RoutedRequest<?> routerRequest = new RoutedRequest<>(routeResult, request);

	Attribute<FileUploads> attribute = new SimpleAttribute();
	attribute.set(new FileUploads(dir));
	Channel channel = mock(Channel.class);
	when(channel.attr(any(AttributeKey.class))).thenReturn(attribute);

	ChannelHandlerContext context = mock(ChannelHandlerContext.class);
	when(context.channel()).thenReturn(channel);

	handler.respondAsLeader(context, routerRequest, mockRestfulGateway);

	// the (asynchronous) request processing is not yet complete so the files should still exist
	Assert.assertTrue(Files.exists(file));
	requestProcessingCompleteFuture.complete(null);
	Assert.assertFalse(Files.exists(file));
}
 

private void handleException(Channel channel, Throwable cause) throws IOException {
	LOG.error("Encountered error while consuming partitions", cause);

	fatalError = true;
	releaseAllResources();

	if (channel.isActive()) {
		channel.writeAndFlush(new ErrorResponse(cause)).addListener(ChannelFutureListener.CLOSE);
	}
}
 

/**
 * Tries to write&flush unannounced credits for the next input channel in queue.
 *
 * <p>This method may be called by the first input channel enqueuing, or the complete
 * future's callback in previous input channel, or the channel writability changed event.
 */
private void writeAndFlushNextMessageIfPossible(Channel channel) {
	if (channelError.get() != null || !channel.isWritable()) {
		return;
	}

	while (true) {
		RemoteInputChannel inputChannel = inputChannelsWithCredit.poll();

		// The input channel may be null because of the write callbacks
		// that are executed after each write.
		if (inputChannel == null) {
			return;
		}

		//It is no need to notify credit for the released channel.
		if (!inputChannel.isReleased()) {
			AddCredit msg = new AddCredit(
				inputChannel.getPartitionId(),
				inputChannel.getAndResetUnannouncedCredit(),
				inputChannel.getInputChannelId());

			// Write and flush and wait until this is done before
			// trying to continue with the next input channel.
			channel.writeAndFlush(msg).addListener(writeListener);

			return;
		}
	}
}
 

/**
 * Get the server port on which the artifact server is listening.
 */
public synchronized int getServerPort() {
	Channel server = this.serverChannel;
	if (server != null) {
		try {
			return ((InetSocketAddress) server.localAddress()).getPort();
		} catch (Exception e) {
			LOG.error("Cannot access local server port", e);
		}
	}
	return -1;
}
 

public static ChannelFuture flush(Channel ch, HttpRequest req, HttpResponse res) {
	if (!HttpHeaders.isKeepAlive(req)) {
		return ch.writeAndFlush(res).addListener(ChannelFutureListener.CLOSE);
	} else {
		res.headers().set(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
		return ch.writeAndFlush(res);
	}
}