Netty使用案例 -堆外内存泄漏跟踪
内存池ByteBuf泄漏没有引起堆内存溢出
服务端代码入下
public class RouterServerHandler extends ChannelInboundHandlerAdapter {
static ExecutorService executorService = Executors.newSingleThreadExecutor();
PooledByteBufAllocator allocator = new PooledByteBufAllocator(false);
@Override
public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
ByteBuf regMsg = (ByteBuf) msg;
byte[] body = new byte[regMsg.readableBytes()];
executorService.execute(new Runnable() {
@Override
public void run() {
ByteBuf respMsg = allocator.heapBuffer(body.length);
//将请求直接转化为响应
respMsg.writeBytes(body);
ctx.writeAndFlush(respMsg);
//ctx.fireChannelRead(msg);
}
});
}
@Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
cause.printStackTrace();
ctx.close();
}
}
public class RouterServer {
public static void main(String[] args) throws InterruptedException {
EventLoopGroup bossGroup = new NioEventLoopGroup(1);
EventLoopGroup workerGroup = new NioEventLoopGroup();
try {
ServerBootstrap serverBootstrap = new ServerBootstrap();
serverBootstrap.group(bossGroup, workerGroup)
.channel(NioServerSocketChannel.class)
.option(ChannelOption.SO_BACKLOG, 100)
.handler(new LoggingHandler(LogLevel.INFO))
.childHandler(new ChannelInitializer<SocketChannel>() {
@Override
protected void initChannel(SocketChannel ch) throws Exception {
ChannelPipeline pipeline = ch.pipeline();
pipeline.addLast(new RouterServerHandler());
}
});
//启动服务
ChannelFuture f = serverBootstrap.bind(8080).sync();
//阻塞知道服务关闭
f.channel().closeFuture().sync();
} finally {
bossGroup.shutdownGracefully();
workerGroup.shutdownGracefully();
}
}
}
客户端代码如下
public class RouterClinetHandler extends ChannelInboundHandlerAdapter {
private final ByteBuf firstMessage;
public RouterClinetHandler() {
System.out.println("RouterClinetHandler=");
this.firstMessage = Unpooled.buffer(1024);
for (int i = 0; i < firstMessage.capacity(); i++) {
firstMessage.writeByte((byte)i);
}
}
@Override
public void channelActive(ChannelHandlerContext ctx) throws Exception {
ctx.writeAndFlush(firstMessage);
}
@Override
public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
System.out.println("channelRead=");
ctx.writeAndFlush(msg);
}
@Override
public void channelReadComplete(ChannelHandlerContext ctx) throws Exception {
ctx.flush();
}
@Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
cause.printStackTrace();
ctx.close();
}
}
public class RouterClient {
public static void main(String[] args) throws InterruptedException {
//客户端访问服务
EventLoopGroup group = new NioEventLoopGroup();
try {
Bootstrap b = new Bootstrap();
b.group(group)
.channel(NioSocketChannel.class)
.option(ChannelOption.TCP_NODELAY, true)
.handler(new ChannelInitializer<SocketChannel>() {
@Override
protected void initChannel(SocketChannel ch) throws Exception {
ChannelPipeline p = ch.pipeline();
p.addLast(new RouterClinetHandler());
}
});
// Start the client.
ChannelFuture f = b.connect(String.valueOf(args[0]), Integer.valueOf(args[1])).sync();
f.channel().closeFuture().sync();
} finally {
group.shutdownGracefully();
}
}
}
运行一段时间后server的效果
ByteBuf内存没有释放
对于ByteBuf申请相关代码进行排查，发现响应消息由业务线程创建，但却没有主动释放，因此是响应消息没有释放导致内存泄漏，但是图像看不出对内存泄漏。
PooledUnsafeHeapByteBuf并没有想象中的那么多，最多的是PooledUnsafeDirectByteBuf
业务内存快照
业务从内存池中申请了ByteBuf,但是没有主动释放，最后也没有发生内存泄漏，为什么？
如果是堆内存泄漏
//代码（AbstractNioByteChannel类）
@Override
protected final Object filterOutboundMessage(Object msg) {
if (msg instanceof ByteBuf) {
ByteBuf buf = (ByteBuf) msg;
if (buf.isDirect()) {
return msg;
}
//这里创建DirectBuffer
return newDirectBuffer(buf);
}
if (msg instanceof FileRegion) {
return msg;
}
throw new UnsupportedOperationException(
"unsupported message type: " + StringUtil.simpleClassName(msg) + EXPECTED_TYPES);
}
//代码（AbstractNioChannel类）
protected final ByteBuf newDirectBuffer(ByteBuf buf) {
final int readableBytes = buf.readableBytes();
if (readableBytes == 0) {
ReferenceCountUtil.safeRelease(buf);
return Unpooled.EMPTY_BUFFER;
}
final ByteBufAllocator alloc = alloc();
if (alloc.isDirectBufferPooled()) {
//将HeapByteBuffer转换成DirectByteBuffer
ByteBuf directBuf = alloc.directBuffer(readableBytes);
directBuf.writeBytes(buf, buf.readerIndex(), readableBytes);
//释放PooledHeapByteBuf到内存池
ReferenceCountUtil.safeRelease(buf);
return directBuf;
}
}
。。。
如果消息完整地被写到SocketChannel中，则释放DirectByteBuffer，代码（ChannelOutboundBuffer类）：
public boolean remove() {
Entry e = flushedEntry;
if (e == null) {
clearNioBuffers();
return false;
}
Object msg = e.msg;
ChannelPromise promise = e.promise;
int size = e.pendingSize;
removeEntry(e);
if (!e.cancelled) {
ReferenceCountUtil.safeRelease(msg);
safeSuccess(promise);
decrementPendingOutboundBytes(size, false, true);
}
//后续代码省略
}
代码跟踪
- 设置断点到发送消息处writeAndFlush，获取到的PooledUnsafeHeapByteBuf实例的ID为2021
- 在HeapByteBuffer转换成DirectByteBuffer处设置断点，发现实例ID为2021的PooledUnsafeHeapByteBuf被释放，在AbstractNioChannel.newDirectBuffer(buf)中的ReferenceCountUtil.safeRelease(buf)
- 转换之后待发送的响应消息PooledUnsafeDirectByteBuf实例的ID为2049，AbstractNioChannel.newDirectBuffer中的ByteBuf directBuf = alloc.directBuffer(readableBytes);
- 在响应消息发送完成后，实例ID为2049的PooledUnsafeDirectByteBuf被释放到内存池中
以上跟踪确认不是响应消息没有主动释放导致的内存泄漏。
堆外内存泄漏
对请求消息的内存分配分析，发现NioByteUnsafe的read方法中申请了内存(NioByteUnsafe类)
//NioByteUnsaf.read
byteBuf = allocHandle.allocate(allocator);
//继续对allocate方法进行分析，发现调用的是DefaultMaxMessagesRecvByteBuf- Allocator$MaxMessageHandle
public ByteBuf allocate(ByteBufAllocator alloc) {
return alloc.ioBuffer(guess());
}
alloc.ioBuffer方法最终会调用PooledByteBufAllocator的newDirectBuffer方法创建PooledDirectByteBuf对象。
请求ByteBuf的创建分析完,由于业务的RouterServerHandler继承自ChannelInboundHandlerAdapter，它的channelRead(ChannelHandlerContext ctx, Object msg)方法执行完成，ChannelHandler的执行就结束了,请求ByteBuf被Netty框架申请后竟然没有被释放,
为了验证分析，在业务代码中调用ReferenceCountUtil的release方法进行手动对内存释放操作。代码调整添加
ByteBuf regMsg = (ByteBuf) msg;
byte[] body = new byte[regMsg.readableBytes()];
//添加释放操作
ReferenceCountUtil.release(regMsg);
。。。
修改之后继续进行压测，发现系统运行平稳，没有发生OutOfDirectMemoryError异常。对内存活动对象进行排序，没有再发现大量的PoolChunk对象，内存泄漏问题解决。
理解ByteBuf申请和释放
PooledDirectByteBuf和PooledHeapByteBuf：由Netty的NioEventLoop线程在处理Channel的读操作时分配，需要在业务ChannelInboundHandler处理完请求消息之后释放（通常在解码之后），它的释放有两种策略。
- 策略1 ：业务ChannelInboundHandler继承自SimpleChannelInboundHandler，实现它的抽象方法channelRead0(ChannelHandlerContext ctx, I msg)，ByteBuf的释放业务不用关心，由SimpleChannelInboundHandler负责释放（SimpleChannelInboundHandler类）
public void channelRead(ChannelHandlerContext ctx, Object msg) throws
Exception {
boolean release = true;
try {
if (acceptInboundMessage(msg)) {
I imsg = (I) msg;
//调用channelRead0之后执行ReferenceCountUtil.release(msg)释放当前请求消息
channelRead0(ctx, imsg);
} else {
release = false;
//如果没有匹配上需要继续执行后续的ChannelInboundHandler，则不释放当前请求消息，调用ctx.fireChannelRead(msg)驱动ChannelPipeline继续执行。
ctx.fireChannelRead(msg);
}
} finally {
if (autoRelease && release) {
ReferenceCountUtil.release(msg);
}
}
}
将上边的Handler由继承的ChannelInboundHandlerAdapter调整SimpleChannelInboundHandler，对修改之后的代码做性能测试，发现内存占用平稳，无内存泄漏问题。
public class SimpServerHandler extends SimpleChannelInboundHandler<ByteBuf> {
static ExecutorService executorService = Executors.newSingleThreadExecutor();
PooledByteBufAllocator allocator = new PooledByteBufAllocator(false);
@Override
protected void channelRead0(ChannelHandlerContext ctx, ByteBuf msg) throws Exception {
ByteBuf regMsg = (ByteBuf) msg;
byte[] body = new byte[msg.readableBytes()];
//注意这里删除要不会出现onUnhandledInboundException
//ReferenceCountUtil.release(regMsg);
executorService.execute(new Runnable() {
@Override
public void run() {
ByteBuf respMsg = allocator.heapBuffer(body.length);
//将请求直接转化为响应
respMsg.writeBytes(body);
ctx.writeAndFlush(respMsg);
//ctx.fireChannelRead(msg);
}
});
}
}
- 策略2 在业务ChannelInboundHandler中调用ctx.fireChannelRead(msg)方法，让请求消息继续向后执行，直到调用DefaultChannelPipeline的内部类TailContext，由它来负责释放请求消息。
protected void onUnhandledInboundMessage(Object msg) {
try {
logger.debug(
"Discarded inbound message {} that reached at the tail of the pipeline. " +
"Please check your pipeline configuration.", msg);
} finally {
ReferenceCountUtil.release(msg);
}
}
//调整之前的代码RouterServerHandler
//ctx.writeAndFlush(respMsg);修改为以下
ctx.fireChannelRead(msg);
同样使用后运行正常。
基于ByteBuf请求和响应创建
基于非内存请求
//业务使用非内存池模式覆盖Netty默认的内存池模式创建请求ByteBuf，按照内存池的方式释放内存
.childHandler(new ChannelInitializer<SocketChannel>() {
@Override
public void initChannel(SocketChannel ch) throws Exception {
ChannelPipeline p = ch.pipeline();
//设置非内存池请求
ch.config().setAllocator(UnpooledByteBufAllocator.DEFAULT);
p.addLast(new RouterServerHandler());
}
});
}
基于内存池响应
只要调用了writeAndFlush或者flush方法，在消息发送完成后都会由Netty框架进行内存释放，业务不需要主动释放内存。
基于非内存池响应
无论是基于内存池还是非内存池分配的ByteBuf，如果是堆内存，则将堆内存转换成堆外内存，然后释放HeapByteBuffer，待消息发送完成，再释放转换后的DirectByteBuf；如果是DirectByteBuffer，则不需要转换，待消息发送完成之后释放。因此对于需要发送的响应ByteBuf，由业务创建，但是不需要由业务来释放。
解读《Netty进阶之路：跟着案例学Netty》-Netty内存池泄漏疑云案例