org.apache.flink.shaded.netty4.io.netty.channel.ChannelHandlerContext Java Examples

@Override
protected ByteBuf extractFrame(ChannelHandlerContext ctx, ByteBuf buffer, int index, int length) {
	if (restoreOldNettyBehaviour) {
		/*
		 * For non-credit based code paths with Netty >= 4.0.28.Final:
		 * These versions contain an improvement by Netty, which slices a Netty buffer
		 * instead of doing a memory copy [1] in the
		 * LengthFieldBasedFrameDecoder. In some situations, this
		 * interacts badly with our Netty pipeline leading to OutOfMemory
		 * errors.
		 *
		 * [1] https://github.com/netty/netty/issues/3704
		 *
		 * TODO: remove along with the non-credit based fallback protocol
		 */
		ByteBuf frame = ctx.alloc().buffer(length);
		frame.writeBytes(buffer, index, length);
		return frame;
	} else {
		return super.extractFrame(ctx, buffer, index, length);
	}
}
 

@Override
public void channelActive(ChannelHandlerContext ctx) throws Exception {
	final Channel ch = ctx.channel();

	assertEquals("Low watermark", expectedLowWatermark, ch.config().getWriteBufferLowWaterMark());
	assertEquals("High watermark", expectedHighWatermark, ch.config().getWriteBufferHighWaterMark());

	// Start with a writable channel
	assertTrue(ch.isWritable());

	// First buffer should not change writability
	ch.write(buffer());
	assertTrue(ch.isWritable());

	// ...second buffer should though
	ch.write(buffer());
	assertFalse(ch.isWritable());

	// Flush everything and close the channel after the writability changed event is fired.
	hasFlushed = true;
	ch.flush();
}
 

/**
 * Verifies that {@link RemoteInputChannel#onError(Throwable)} is called when a
 * {@link BufferResponse} is received but no available buffer in input channel.
 */
@Test
public void testThrowExceptionForNoAvailableBuffer() throws Exception {
	final SingleInputGate inputGate = createSingleInputGate(1);
	final RemoteInputChannel inputChannel = spy(InputChannelBuilder.newBuilder().buildRemoteChannel(inputGate));

	final CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
	handler.addInputChannel(inputChannel);

	assertEquals("There should be no buffers available in the channel.",
			0, inputChannel.getNumberOfAvailableBuffers());

	final BufferResponse bufferResponse = createBufferResponse(
		TestBufferFactory.createBuffer(TestBufferFactory.BUFFER_SIZE),
		0,
		inputChannel.getInputChannelId(),
		2,
		new NetworkBufferAllocator(handler));
	assertNull(bufferResponse.getBuffer());

	handler.channelRead(mock(ChannelHandlerContext.class), bufferResponse);
	verify(inputChannel, times(1)).onError(any(IllegalStateException.class));
}
 

/**
 * Tests a fix for FLINK-1627.
 *
 * <p> FLINK-1627 discovered a race condition, which could lead to an infinite loop when a
 * receiver was cancelled during a certain time of decoding a message. The test reproduces the
 * input, which lead to the infinite loop: when the handler gets a reference to the buffer
 * provider of the receiving input channel, but the respective input channel is released (and
 * the corresponding buffer provider destroyed), the handler did not notice this.
 *
 * @see <a href="https://issues.apache.org/jira/browse/FLINK-1627">FLINK-1627</a>
 */
@Test(timeout = 60000)
@SuppressWarnings("unchecked")
public void testReleaseInputChannelDuringDecode() throws Exception {
	// Mocks an input channel in a state as it was released during a decode.
	final BufferProvider bufferProvider = mock(BufferProvider.class);
	when(bufferProvider.requestBuffer()).thenReturn(null);
	when(bufferProvider.isDestroyed()).thenReturn(true);
	when(bufferProvider.addBufferListener(any(BufferListener.class))).thenReturn(false);

	final RemoteInputChannel inputChannel = mock(RemoteInputChannel.class);
	when(inputChannel.getInputChannelId()).thenReturn(new InputChannelID());
	when(inputChannel.getBufferProvider()).thenReturn(bufferProvider);

	final BufferResponse receivedBuffer = createBufferResponse(
			TestBufferFactory.createBuffer(TestBufferFactory.BUFFER_SIZE), 0, inputChannel.getInputChannelId(), 2);

	final PartitionRequestClientHandler client = new PartitionRequestClientHandler();
	client.addInputChannel(inputChannel);

	client.channelRead(mock(ChannelHandlerContext.class), receivedBuffer);
}
 

@Override
public void channelActive(ChannelHandlerContext ctx) throws Exception {
	final Channel ch = ctx.channel();

	assertEquals("Low watermark", expectedLowWatermark, ch.config().getWriteBufferLowWaterMark());
	assertEquals("High watermark", expectedHighWatermark, ch.config().getWriteBufferHighWaterMark());

	// Start with a writable channel
	assertTrue(ch.isWritable());

	// First buffer should not change writability
	ch.write(buffer());
	assertTrue(ch.isWritable());

	// ...second buffer should though
	ch.write(buffer());
	assertFalse(ch.isWritable());

	// Flush everything and close the channel after the writability changed event is fired.
	hasFlushed = true;
	ch.flush();
}
 

@Override
protected void channelRead0(ChannelHandlerContext channelHandlerContext, HttpRequest httpRequest) {
	if (HttpHeaders.is100ContinueExpected(httpRequest)) {
		channelHandlerContext.writeAndFlush(new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.CONTINUE));
		return;
	}

	// Route
	HttpMethod method = httpRequest.getMethod();
	QueryStringDecoder qsd = new QueryStringDecoder(httpRequest.uri());
	RouteResult<?> routeResult = router.route(method, qsd.path(), qsd.parameters());

	if (routeResult == null) {
		respondNotFound(channelHandlerContext, httpRequest);
		return;
	}

	routed(channelHandlerContext, routeResult, httpRequest);
}
 

@Override
protected void decode(
		final ChannelHandlerContext ctx,
		final HttpObject msg,
		final List<Object> out) throws Exception {

	try {
		super.decode(ctx, msg, out);
	} catch (final TooLongFrameException e) {
		HandlerUtils.sendErrorResponse(
			ctx,
			false,
			new ErrorResponseBody(String.format(
				e.getMessage() + " Try to raise [%s]",
				RestOptions.SERVER_MAX_CONTENT_LENGTH.key())),
			HttpResponseStatus.REQUEST_ENTITY_TOO_LARGE,
			responseHeaders);
	}
}
 

/**
 * Tests a fix for FLINK-1761.
 *
 * <p>FLINK-1761 discovered an IndexOutOfBoundsException, when receiving buffers of size 0.
 */
@Test
public void testReceiveEmptyBuffer() throws Exception {
	// Minimal mock of a remote input channel
	final BufferProvider bufferProvider = mock(BufferProvider.class);
	when(bufferProvider.requestBuffer()).thenReturn(TestBufferFactory.createBuffer(0));

	final RemoteInputChannel inputChannel = mock(RemoteInputChannel.class);
	when(inputChannel.getInputChannelId()).thenReturn(new InputChannelID());
	when(inputChannel.getBufferProvider()).thenReturn(bufferProvider);

	// An empty buffer of size 0
	final Buffer emptyBuffer = TestBufferFactory.createBuffer(0);

	final CreditBasedPartitionRequestClientHandler client = new CreditBasedPartitionRequestClientHandler();
	client.addInputChannel(inputChannel);

	final int backlog = 2;
	final BufferResponse receivedBuffer = createBufferResponse(
		emptyBuffer,
		0,
		inputChannel.getInputChannelId(),
		backlog,
		new NetworkBufferAllocator(client));

	// Read the empty buffer
	client.channelRead(mock(ChannelHandlerContext.class), receivedBuffer);

	// This should not throw an exception
	verify(inputChannel, never()).onError(any(Throwable.class));
	verify(inputChannel, times(1)).onEmptyBuffer(0, backlog);
}
 

/**
 * Verifies that {@link RemoteInputChannel#onBuffer(Buffer, int, int)} is called when a
 * {@link BufferResponse} is received.
 */
@Test
public void testReceiveBuffer() throws Exception {
	final NetworkBufferPool networkBufferPool = new NetworkBufferPool(10, 32, 2);
	final SingleInputGate inputGate = createSingleInputGate(1);
	final RemoteInputChannel inputChannel = InputChannelBuilder.newBuilder()
		.setMemorySegmentProvider(networkBufferPool)
		.buildRemoteAndSetToGate(inputGate);
	try {
		final BufferPool bufferPool = networkBufferPool.createBufferPool(8, 8);
		inputGate.setBufferPool(bufferPool);
		inputGate.assignExclusiveSegments();

		final PartitionRequestClientHandler handler = new PartitionRequestClientHandler();
		handler.addInputChannel(inputChannel);

		final int backlog = 2;
		final BufferResponse bufferResponse = createBufferResponse(
			TestBufferFactory.createBuffer(32), 0, inputChannel.getInputChannelId(), backlog);
		handler.channelRead(mock(ChannelHandlerContext.class), bufferResponse);

		assertEquals(1, inputChannel.getNumberOfQueuedBuffers());
	} finally {
		// Release all the buffer resources
		inputGate.close();

		networkBufferPool.destroyAllBufferPools();
		networkBufferPool.destroy();
	}
}
 

private void handleSsl(ChannelHandlerContext context) {
	SslHandler sslHandler = sslHandlerFactory.createNettySSLHandler();
	try {
		context.pipeline().replace(this, SSL_HANDLER_NAME, sslHandler);
	} catch (Throwable t) {
		ReferenceCountUtil.safeRelease(sslHandler.engine());
		throw t;
	}
}
 

/**
 * Verifies that {@link RemoteInputChannel#onFailedPartitionRequest()} is called when a
 * {@link PartitionNotFoundException} is received.
 */
@Test
public void testReceivePartitionNotFoundException() throws Exception {
	// Minimal mock of a remote input channel
	final BufferProvider bufferProvider = mock(BufferProvider.class);
	when(bufferProvider.requestBuffer()).thenReturn(TestBufferFactory.createBuffer(0));

	final RemoteInputChannel inputChannel = mock(RemoteInputChannel.class);
	when(inputChannel.getInputChannelId()).thenReturn(new InputChannelID());
	when(inputChannel.getBufferProvider()).thenReturn(bufferProvider);

	final ErrorResponse partitionNotFound = new ErrorResponse(
		new PartitionNotFoundException(new ResultPartitionID()),
		inputChannel.getInputChannelId());

	final CreditBasedPartitionRequestClientHandler client = new CreditBasedPartitionRequestClientHandler();
	client.addInputChannel(inputChannel);

	// Mock channel context
	ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);
	when(ctx.channel()).thenReturn(mock(Channel.class));

	client.channelActive(ctx);

	client.channelRead(ctx, partitionNotFound);

	verify(inputChannel, times(1)).onFailedPartitionRequest();
}
 

@Override
public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
	try {
		ByteBuf buf = (ByteBuf) msg;
		MessageType msgType = MessageSerializer.deserializeHeader(buf);

		if (msgType == MessageType.REQUEST_RESULT) {
			long requestId = MessageSerializer.getRequestId(buf);
			RESP result = serializer.deserializeResponse(buf);
			callback.onRequestResult(requestId, result);
		} else if (msgType == MessageType.REQUEST_FAILURE) {
			RequestFailure failure = MessageSerializer.deserializeRequestFailure(buf);
			callback.onRequestFailure(failure.getRequestId(), failure.getCause());
		} else if (msgType == MessageType.SERVER_FAILURE) {
			throw MessageSerializer.deserializeServerFailure(buf);
		} else {
			throw new IllegalStateException("Unexpected response type '" + msgType + "'");
		}
	} catch (Throwable t1) {
		try {
			callback.onFailure(t1);
		} catch (Throwable t2) {
			LOG.error("Failed to notify callback about failure", t2);
		}
	} finally {
		ReferenceCountUtil.release(msg);
	}
}
 

@Override
public void channelInactive(ChannelHandlerContext ctx) throws Exception {
	// Unexpected close. In normal operation, the client closes the connection after all input
	// channels have been removed. This indicates a problem with the remote task manager.
	if (!inputChannels.isEmpty()) {
		final SocketAddress remoteAddr = ctx.channel().remoteAddress();

		notifyAllChannelsOfErrorAndClose(new RemoteTransportException(
				"Connection unexpectedly closed by remote task manager '" + remoteAddr + "'. "
						+ "This might indicate that the remote task manager was lost.",
				remoteAddr));
	}

	super.channelInactive(ctx);
}
 

public static ChannelFuture flush(ChannelHandlerContext ctx, HttpRequest request, HttpResponse response) {
	if (!HttpHeaders.isKeepAlive(request)) {
		return ctx.writeAndFlush(response).addListener(ChannelFutureListener.CLOSE);
	} else {
		response.headers().set(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
		return ctx.writeAndFlush(response);
	}
}
 

/**
 * Triggered by notifying credit available in the client handler pipeline.
 *
 * <p>Enqueues the input channel and will trigger write&flush unannounced credits
 * for this input channel if it is the first one in the queue.
 */
@Override
public void userEventTriggered(ChannelHandlerContext ctx, Object msg) throws Exception {
	if (msg instanceof RemoteInputChannel) {
		boolean triggerWrite = inputChannelsWithCredit.isEmpty();

		inputChannelsWithCredit.add((RemoteInputChannel) msg);

		if (triggerWrite) {
			writeAndFlushNextMessageIfPossible(ctx.channel());
		}
	} else {
		ctx.fireUserEventTriggered(msg);
	}
}
 

/**
 * Sends the given error response and status code to the given channel.
 *
 * @param channelHandlerContext identifying the open channel
 * @param keepAlive If the connection should be kept alive.
 * @param errorMessage which should be sent
 * @param statusCode of the message to send
 * @param headers additional header values
 */
public static CompletableFuture<Void> sendErrorResponse(
		ChannelHandlerContext channelHandlerContext,
		boolean keepAlive,
		ErrorResponseBody errorMessage,
		HttpResponseStatus statusCode,
		Map<String, String> headers) {

	StringWriter sw = new StringWriter();
	try {
		mapper.writeValue(sw, errorMessage);
	} catch (IOException e) {
		// this should never happen
		LOG.error("Internal server error. Could not map error response to JSON.", e);
		return sendResponse(
			channelHandlerContext,
			keepAlive,
			"Internal server error. Could not map error response to JSON.",
			HttpResponseStatus.INTERNAL_SERVER_ERROR,
			headers);
	}
	return sendResponse(
		channelHandlerContext,
		keepAlive,
		sw.toString(),
		statusCode,
		headers);
}
 

/**
 * Tests a fix for FLINK-1761.
 *
 * <p>FLINK-1761 discovered an IndexOutOfBoundsException, when receiving buffers of size 0.
 */
@Test
public void testReceiveEmptyBuffer() throws Exception {
	// Minimal mock of a remote input channel
	final BufferProvider bufferProvider = mock(BufferProvider.class);
	when(bufferProvider.requestBuffer()).thenReturn(TestBufferFactory.createBuffer(0));

	final RemoteInputChannel inputChannel = mock(RemoteInputChannel.class);
	when(inputChannel.getInputChannelId()).thenReturn(new InputChannelID());
	when(inputChannel.getBufferProvider()).thenReturn(bufferProvider);

	// An empty buffer of size 0
	final Buffer emptyBuffer = TestBufferFactory.createBuffer(0);

	final int backlog = -1;
	final BufferResponse receivedBuffer = createBufferResponse(
		emptyBuffer, 0, inputChannel.getInputChannelId(), backlog);

	final PartitionRequestClientHandler client = new PartitionRequestClientHandler();
	client.addInputChannel(inputChannel);

	// Read the empty buffer
	client.channelRead(mock(ChannelHandlerContext.class), receivedBuffer);

	// This should not throw an exception
	verify(inputChannel, never()).onError(any(Throwable.class));
	verify(inputChannel, times(1)).onEmptyBuffer(0, backlog);
}
 

@Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
	try {
		callback.onFailure(cause);
	} catch (Throwable t) {
		LOG.error("Failed to notify callback about failure", t);
	}
}
 

@Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
	final String msg = "Exception in server pipeline. Caused by: " + ExceptionUtils.stringifyException(cause);
	final ByteBuf err = MessageSerializer.serializeServerFailure(ctx.alloc(), new RuntimeException(msg));

	LOG.debug(msg);
	ctx.writeAndFlush(err).addListener(ChannelFutureListener.CLOSE);
}
 

/**
 * Verifies that {@link RemoteInputChannel#onFailedPartitionRequest()} is called when a
 * {@link PartitionNotFoundException} is received.
 */
@Test
public void testReceivePartitionNotFoundException() throws Exception {
	// Minimal mock of a remote input channel
	final BufferProvider bufferProvider = mock(BufferProvider.class);
	when(bufferProvider.requestBuffer()).thenReturn(TestBufferFactory.createBuffer(0));

	final RemoteInputChannel inputChannel = mock(RemoteInputChannel.class);
	when(inputChannel.getInputChannelId()).thenReturn(new InputChannelID());
	when(inputChannel.getBufferProvider()).thenReturn(bufferProvider);

	final ErrorResponse partitionNotFound = new ErrorResponse(
		new PartitionNotFoundException(new ResultPartitionID()),
		inputChannel.getInputChannelId());

	final PartitionRequestClientHandler client = new PartitionRequestClientHandler();
	client.addInputChannel(inputChannel);

	// Mock channel context
	ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);
	when(ctx.channel()).thenReturn(mock(Channel.class));

	client.channelActive(ctx);

	client.channelRead(ctx, partitionNotFound);

	verify(inputChannel, times(1)).onFailedPartitionRequest();
}
 

/**
 * Send the "304 Not Modified" response. This response can be used when the
 * file timestamp is the same as what the browser is sending up.
 *
 * @param ctx The channel context to write the response to.
 */
public static void sendNotModified(ChannelHandlerContext ctx) {
	FullHttpResponse response = new DefaultFullHttpResponse(HTTP_1_1, NOT_MODIFIED);
	setDateHeader(response);

	// close the connection as soon as the error message is sent.
	ctx.writeAndFlush(response).addListener(ChannelFutureListener.CLOSE);
}
 

/**
 * Writes a simple  error response message.
 *
 * @param ctx    The channel context to write the response to.
 * @param status The response status.
 */
private static void sendError(ChannelHandlerContext ctx, HttpResponseStatus status) {
	FullHttpResponse response = new DefaultFullHttpResponse(
		HTTP_1_1, status, Unpooled.copiedBuffer("Failure: " + status + "\r\n", CharsetUtil.UTF_8));
	HttpHeaders.setHeader(response, CONTENT_TYPE, "text/plain; charset=UTF-8");

	// close the connection as soon as the error message is sent.
	ctx.writeAndFlush(response).addListener(ChannelFutureListener.CLOSE);
}
 

@Override
public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
	HttpRequest request = msg instanceof HttpRequest ? (HttpRequest) msg : null;
	String path = request == null ? "" : request.uri();
	String redirectAddress = getRedirectAddress(ctx);
	log.trace("Received non-SSL request, redirecting to {}{}", redirectAddress, path);
	HttpResponse response = HandlerRedirectUtils.getRedirectResponse(
		redirectAddress, path, HttpResponseStatus.MOVED_PERMANENTLY);
	if (request != null) {
		KeepAliveWrite.flush(ctx, request, response);
	} else {
		ctx.writeAndFlush(response).addListener(ChannelFutureListener.CLOSE);
	}
}
 

@Override
public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
	try {
		decodeMsg(msg);
	} catch (Throwable t) {
		notifyAllChannelsOfErrorAndClose(t);
	}
}
 

/**
 * Triggered by notifying credit available in the client handler pipeline.
 *
 * <p>Enqueues the input channel and will trigger write&flush unannounced credits
 * for this input channel if it is the first one in the queue.
 */
@Override
public void userEventTriggered(ChannelHandlerContext ctx, Object msg) throws Exception {
	if (msg instanceof RemoteInputChannel) {
		boolean triggerWrite = inputChannelsWithCredit.isEmpty();

		inputChannelsWithCredit.add((RemoteInputChannel) msg);

		if (triggerWrite) {
			writeAndFlushNextMessageIfPossible(ctx.channel());
		}
	} else {
		ctx.fireUserEventTriggered(msg);
	}
}
 

/**
 * Send the "405 Method Not Allowed" response.
 *
 * @param ctx The channel context to write the response to.
 */
private static void sendMethodNotAllowed(ChannelHandlerContext ctx) {
	FullHttpResponse response = new DefaultFullHttpResponse(HTTP_1_1, METHOD_NOT_ALLOWED);

	// close the connection as soon as the error message is sent.
	ctx.writeAndFlush(response).addListener(ChannelFutureListener.CLOSE);
}
 

@Override
public void channelRegistered(final ChannelHandlerContext ctx) throws Exception {
	if (this.ctx == null) {
		this.ctx = ctx;
	}

	super.channelRegistered(ctx);
}
 

@Override
protected void channelRead0(ChannelHandlerContext ctx, RoutedRequest routed) throws Exception {
	HttpResponse response = new DefaultFullHttpResponse(
		HttpVersion.HTTP_1_1, HttpResponseStatus.OK, Unpooled.wrappedBuffer(encodedText));

	response.headers().set(HttpHeaders.Names.CONTENT_LENGTH, encodedText.length);
	response.headers().set(HttpHeaders.Names.CONTENT_TYPE, "text/plain");

	KeepAliveWrite.flush(ctx, routed.getRequest(), response);
}
 

/**
 * Send the "304 Not Modified" response. This response can be used when the
 * file timestamp is the same as what the browser is sending up.
 *
 * @param ctx The channel context to write the response to.
 */
public static void sendNotModified(ChannelHandlerContext ctx) {
	FullHttpResponse response = new DefaultFullHttpResponse(HTTP_1_1, NOT_MODIFIED);
	setDateHeader(response);

	// close the connection as soon as the error message is sent.
	ctx.writeAndFlush(response).addListener(ChannelFutureListener.CLOSE);
}
 

@Override
public void channelWritabilityChanged(ChannelHandlerContext ctx) throws Exception {
	if (hasFlushed) {
		// After flushing the writability should change back to writable
		assertTrue(ctx.channel().isWritable());

		// Close the channel. This will terminate the main test Thread.
		ctx.close();
	}

	super.channelWritabilityChanged(ctx);
}