ZooKeeper源码分析之NettyServerCnxnFactory
文章目录
- 2021SC@SDUSC
- 属性
- 内部类
- 构造方法
- 总结
2021SC@SDUSC
NettyServerCnxnFactory使用netty进行网络IO。其中，NettyServerCnxnFactory与NIOServerCnxnFactory的configure()，startup()以及start()类似，在此文章中不再阐述。
属性
维护客户端与服务器之间基于Netty创建的连接属性。
//日志
private static final Logger LOG = LoggerFactory.getLogger(NettyServerCnxnFactory.class);
/**
* 允许客户端服务器套接字同时接受SSL和明文连接
*/
public static final String PORT_UNIFICATION_KEY = "zookeeper.client.portUnification";
//判断是否使用同一端口
private final boolean shouldUsePortUnification;
private static final byte TLS_HANDSHAKE_RECORD_TYPE = 0x16;
//连接三次握手
private final AtomicInteger outstandingHandshake = new AtomicInteger();
//三次握手限制
public static final String OUTSTANDING_HANDSHAKE_LIMIT = "zookeeper.netty.server.outstandingHandshake.limit";
private int outstandingHandshakeLimit;
private boolean handshakeThrottlingEnabled;
//服务器引导
private final ServerBootstrap bootstrap;
//父信道
private Channel parentChannel;
//所有的连接信道
private final ChannelGroup allChannels = new DefaultChannelGroup("zkServerCnxns", new DefaultEventExecutor());
private final Map<InetAddress, AtomicInteger> ipMap = new ConcurrentHashMap<>();
//本地地址
private InetSocketAddress localAddress;
//最大客户端连接数
private int maxClientCnxns = 60;
//监听存储
int listenBacklog = -1;
//客户端X509单元
private final ClientX509Util x509Util;
//netty超流量控制
public static final String NETTY_ADVANCED_FLOW_CONTROL = "zookeeper.netty.advancedFlowControl.enabled";
//默认为false
private boolean advancedFlowControlEnabled = false;
//连接属性
private static final AttributeKey<NettyServerCnxn> CONNECTION_ATTRIBUTE = AttributeKey.valueOf("NettyServerCnxn");
//测试分配器
private static final AtomicReference<ByteBufAllocator> TEST_ALLOCATOR = new AtomicReference<>(null);
//连接信道处理器
CnxnChannelHandler channelHandler = new CnxnChannelHandler();
//读取发出的跟踪处理器
ReadIssuedTrackingHandler readIssuedTrackingHandler = new ReadIssuedTrackingHandler();
内部类
NettyServerCnxnFactory中存在两个普通类、一个常量类、和一个静态类。
DualModeSslHandler普通类，保证连接信道的安全性
/**
* 双模式SSL处理器
* 用于检测客户端是否希望使用TLS并以实例形式响应
* 首先检查静态TLS标头的第一个字节以进行确定，然后将其放回流中，并实例化正确的ChannelHandler
*/
class DualModeSslHandler extends OptionalSslHandler {
DualModeSslHandler(SslContext sslContext) {
super(sslContext);
}
//对ChannelHandler中的信息进行解码
@Override
protected void decode(ChannelHandlerContext context, ByteBuf in, List<Object> out) throws Exception {
if (in.readableBytes() >= 5) {
super.decode(context, in, out);
} else if (in.readableBytes() > 0) {
// 检测正确的ssl连接需要5个字节。
// 在服务器接收较少的情况下，检查是否无法使用明文。
// 对于任何四个字母的work命令，都会发生这种情况。
if (TLS_HANDSHAKE_RECORD_TYPE != in.getByte(0)) {
LOG.debug("first byte {} does not match TLS handshake, failing to plaintext", in.getByte(0));
handleNonSsl(context);
}
}
}
/**
* 直接从可选的SSLHandler拉取，以允许访问
* @param context
*/
private void handleNonSsl(ChannelHandlerContext context) {
ChannelHandler handler = this.newNonSslHandler(context);
if (handler != null) {
context.pipeline().replace(this, this.newNonSslHandlerName(), handler);
} else {
context.pipeline().remove(this);
}
}
@Override
protected SslHandler newSslHandler(ChannelHandlerContext context, SslContext sslContext) {
NettyServerCnxn cnxn = Objects.requireNonNull(context.channel().attr(CONNECTION_ATTRIBUTE).get());
LOG.debug("creating ssl handler for session {}", cnxn.getSessionId());
SslHandler handler = super.newSslHandler(context, sslContext);
Future<Channel> handshakeFuture = handler.handshakeFuture();
handshakeFuture.addListener(new CertificateVerifier(handler, cnxn));
return handler;
}
@Override
protected ChannelHandler newNonSslHandler(ChannelHandlerContext context) {
NettyServerCnxn cnxn = Objects.requireNonNull(context.channel().attr(CONNECTION_ATTRIBUTE).get());
LOG.debug("creating plaintext handler for session {}", cnxn.getSessionId());
// Mark handshake finished if it's a insecure cnxn
updateHandshakeCountIfStarted(cnxn);
allChannels.add(context.channel());
addCnxn(cnxn);
return super.newNonSslHandler(context);
}
}
CnxnChannelHandler普通类，用于处理连接信道各种需求，如信道激活、信道失效、异常捕捉、用户事件触发 、阅读信道内的数据、将数据写入信道中等等。
/**
* 继承 ChannelDuplexHandler
* 由于NettyServerCnxnFactory已经继承了ServerCnxnFactory.
* 通过使其成为内部类，该类可以访问成员变量和方法
*/
@Sharable
class CnxnChannelHandler extends ChannelDuplexHandler {
@Override
public void channelActive(ChannelHandlerContext ctx) throws Exception {
if (LOG.isTraceEnabled()) {
LOG.trace("Channel active {}", ctx.channel());
}
final Channel channel = ctx.channel();
//超过限制连接总数，关闭信道
if (limitTotalNumberOfCnxns()) {
ServerMetrics.getMetrics().CONNECTION_REJECTED.add(1);
channel.close();
return;
}
//本地地址
InetAddress addr = ((InetSocketAddress) channel.remoteAddress()).getAddress();
//客户端连接数量超出最大连接数量时，关闭信道
if (maxClientCnxns > 0 && getClientCnxnCount(addr) >= maxClientCnxns) {
ServerMetrics.getMetrics().CONNECTION_REJECTED.add(1);
LOG.warn("Too many connections from {} - max is {}", addr, maxClientCnxns);
channel.close();
return;
}
//创建netty服务连接
NettyServerCnxn cnxn = new NettyServerCnxn(channel, zkServer, NettyServerCnxnFactory.this);
ctx.channel().attr(CONNECTION_ATTRIBUTE).set(cnxn);
//如果握手限制已启用
if (handshakeThrottlingEnabled) {
int outstandingHandshakesNum = outstandingHandshake.addAndGet(1);
if (outstandingHandshakesNum > outstandingHandshakeLimit) {
outstandingHandshake.addAndGet(-1);
channel.close();
ServerMetrics.getMetrics().TLS_HANDSHAKE_EXCEEDED.add(1);
} else {
cnxn.setHandshakeState(HandshakeState.STARTED);
}
}
//如果连接信道安全
if (secure) {
SslHandler sslHandler = ctx.pipeline().get(SslHandler.class);
Future<Channel> handshakeFuture = sslHandler.handshakeFuture();
handshakeFuture.addListener(new CertificateVerifier(sslHandler, cnxn));
} else if (!shouldUsePortUnification) {//没有使用统一端口
allChannels.add(ctx.channel());
addCnxn(cnxn);
}
if (ctx.channel().pipeline().get(SslHandler.class) == null) {
SocketAddress remoteAddress = cnxn.getChannel().remoteAddress();
if (remoteAddress != null
&& !((InetSocketAddress) remoteAddress).getAddress().isLoopbackAddress()) {
LOG.trace("NettyChannelHandler channelActive: remote={} local={}", remoteAddress, cnxn.getChannel().localAddress());
zkServer.serverStats().incrementNonMTLSRemoteConnCount();
} else {
zkServer.serverStats().incrementNonMTLSLocalConnCount();
}
}
}
//信道失效
@Override
public void channelInactive(ChannelHandlerContext ctx) throws Exception {
if (LOG.isTraceEnabled()) {
LOG.trace("Channel inactive {}", ctx.channel());
}
//关闭连接信道
allChannels.remove(ctx.channel());
NettyServerCnxn cnxn = ctx.channel().attr(CONNECTION_ATTRIBUTE).getAndSet(null);
if (cnxn != null) {
if (LOG.isTraceEnabled()) {
LOG.trace("Channel inactive caused close {}", cnxn);
}
updateHandshakeCountIfStarted(cnxn);
cnxn.close(ServerCnxn.DisconnectReason.CHANNEL_DISCONNECTED);
}
}
//异常捕捉
@Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
LOG.warn("Exception caught", cause);
NettyServerCnxn cnxn = ctx.channel().attr(CONNECTION_ATTRIBUTE).getAndSet(null);
if (cnxn != null) {
LOG.debug("Closing {}", cnxn);
updateHandshakeCountIfStarted(cnxn);
cnxn.close(ServerCnxn.DisconnectReason.CHANNEL_CLOSED_EXCEPTION);
}
}
//用户事件触发
@Override
public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception {
try {
if (evt == NettyServerCnxn.ReadEvent.ENABLE) {//读事件
LOG.debug("Received ReadEvent.ENABLE");
NettyServerCnxn cnxn = ctx.channel().attr(CONNECTION_ATTRIBUTE).get();
if (cnxn != null) {//存在连接信道
if (cnxn.getQueuedReadableBytes() > 0) {
cnxn.processQueuedBuffer();
//如果超流量控制启动并且获取队列可读字节
if (advancedFlowControlEnabled && cnxn.getQueuedReadableBytes() == 0) {
//用户读取信道数据
ctx.read();
LOG.debug("Issued a read after queuedBuffer drained");
}
}
}
//如果超流量控制没有启动
if (!advancedFlowControlEnabled) {
ctx.channel().config().setAutoRead(true);
}
} else if (evt == NettyServerCnxn.ReadEvent.DISABLE) {
LOG.debug("Received ReadEvent.DISABLE");
ctx.channel().config().setAutoRead(false);
}
} finally {
ReferenceCountUtil.release(evt);
}
}
//信道阅读
@Override
public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
try {
if (LOG.isTraceEnabled()) {
LOG.trace("message received called {}", msg);
}
try {
LOG.debug("New message {} from {}", msg, ctx.channel());
NettyServerCnxn cnxn = ctx.channel().attr(CONNECTION_ATTRIBUTE).get();
if (cnxn == null) {
LOG.error("channelRead() on a closed or closing NettyServerCnxn");
} else {
cnxn.processMessage((ByteBuf) msg);
}
} catch (Exception ex) {
LOG.error("Unexpected exception in receive", ex);
throw ex;
}
} finally {
ReferenceCountUtil.release(msg);
}
}
//信道阅读完成
@Override
public void channelReadComplete(ChannelHandlerContext ctx) throws Exception {
if (advancedFlowControlEnabled) {
NettyServerCnxn cnxn = ctx.channel().attr(CONNECTION_ATTRIBUTE).get();
if (cnxn != null && cnxn.getQueuedReadableBytes() == 0 && cnxn.readIssuedAfterReadComplete == 0) {
ctx.read();
LOG.debug("Issued a read since we do not have anything to consume after channelReadComplete");
}
}
ctx.fireChannelReadComplete();
}
// Use a single listener instance to reduce GC
// Note: this listener is only added when LOG.isTraceEnabled() is true,
// so it should not do any work other than trace logging.
private final GenericFutureListener<Future<Void>> onWriteCompletedTracer = (f) -> {
if (LOG.isTraceEnabled()) {
LOG.trace("write success: {}", f.isSuccess());
}
};
//消息写入信道
@Override
public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) throws Exception {
if (LOG.isTraceEnabled()) {
promise.addListener(onWriteCompletedTracer);
}
super.write(ctx, msg, promise);
}
}
CertificateVerifier常量类，用于连接认证，确保连接信道的安全。
//认证器
final class CertificateVerifier implements GenericFutureListener<Future<Channel>> {
private final SslHandler sslHandler;
//netty服务连接
private final NettyServerCnxn cnxn;
CertificateVerifier(SslHandler sslHandler, NettyServerCnxn cnxn) {
this.sslHandler = sslHandler;
this.cnxn = cnxn;
}
/**
* 仅当认证通过身份验证时才允许连接保持打开状态
*/
public void operationComplete(Future<Channel> future) {
updateHandshakeCountIfStarted(cnxn);
//连接信道打开
if (future.isSuccess()) {
LOG.debug("Successful handshake with session 0x{}", Long.toHexString(cnxn.getSessionId()));
SSLEngine eng = sslHandler.engine();
// 若没有要求客户提供证书，则不需要认证
if (eng.getNeedClientAuth() || eng.getWantClientAuth()) {
SSLSession session = eng.getSession();
try {
//设置客户端认证链
cnxn.setClientCertificateChain(session.getPeerCertificates());
} catch (SSLPeerUnverifiedException e) {
if (eng.getNeedClientAuth()) {
// 已请求认证，但不存在
LOG.error("Error getting peer certificates", e);
cnxn.close();
return;
} else {
// 已请求认证，但认证是可选的
// TODO: what auth info should we set on the connection?
final Channel futureChannel = future.getNow();
allChannels.add(Objects.requireNonNull(futureChannel));
addCnxn(cnxn);
return;
}
} catch (Exception e) {
LOG.error("Error getting peer certificates", e);
cnxn.close();
return;
}
String authProviderProp = System.getProperty(x509Util.getSslAuthProviderProperty(), "x509");
X509AuthenticationProvider authProvider = (X509AuthenticationProvider) ProviderRegistry.getProvider(authProviderProp);
if (authProvider == null) {
LOG.error("X509 Auth provider not found: {}", authProviderProp);
cnxn.close(ServerCnxn.DisconnectReason.AUTH_PROVIDER_NOT_FOUND);
return;
}
KeeperException.Code code = authProvider.handleAuthentication(cnxn, null);
if (KeeperException.Code.OK != code) {
zkServer.serverStats().incrementAuthFailedCount();
LOG.error("Authentication failed for session 0x{}", Long.toHexString(cnxn.getSessionId()));
cnxn.close(ServerCnxn.DisconnectReason.SASL_AUTH_FAILURE);
return;
}
}
final Channel futureChannel = future.getNow();
allChannels.add(Objects.requireNonNull(futureChannel));
addCnxn(cnxn);
} else {
zkServer.serverStats().incrementAuthFailedCount();
LOG.error("Unsuccessful handshake with session 0x{}", Long.toHexString(cnxn.getSessionId()));
cnxn.close(ServerCnxn.DisconnectReason.FAILED_HANDSHAKE);
}
}
}
ReadIssuedTrackingHandler静态类，与连接信道处理器类似，对信道内的数据进行读取。
//读取跟踪处理器
@Sharable
static class ReadIssuedTrackingHandler extends ChannelDuplexHandler {
@Override
public void read(ChannelHandlerContext ctx) throws Exception {
NettyServerCnxn cnxn = ctx.channel().attr(CONNECTION_ATTRIBUTE).get();
if (cnxn != null) {
cnxn.readIssuedAfterReadComplete++;
}
//读取连接信道中的数据
ctx.read();
}
@Override
public void channelReadComplete(ChannelHandlerContext ctx) throws Exception {
NettyServerCnxn cnxn = ctx.channel().attr(CONNECTION_ATTRIBUTE).get();
if (cnxn != null) {
cnxn.readIssuedAfterReadComplete = 0;
}
//完成读取连接信道中的数据
ctx.fireChannelReadComplete();
}
}
构造方法
bossgroup处理accept事件,默认启动1个线程。
启动workgroup处理网络IO,默认启动核心数∗2个线程。
NettyServerCnxnFactory() {
//客户端x509单元
x509Util = new ClientX509Util();
//使用统一端口
boolean usePortUnification = Boolean.getBoolean(PORT_UNIFICATION_KEY);
LOG.info("{}={}", PORT_UNIFICATION_KEY, usePortUnification);
//如果使用了统一端口
if (usePortUnification) {
try {
//集群配置
QuorumPeerConfig.configureSSLAuth();
} catch (QuorumPeerConfig.ConfigException e) {
LOG.error("unable to set up SslAuthProvider, turning off client port unification", e);
usePortUnification = false;
}
}
this.shouldUsePortUnification = usePortUnification;
this.advancedFlowControlEnabled = Boolean.getBoolean(NETTY_ADVANCED_FLOW_CONTROL);
LOG.info("{} = {}", NETTY_ADVANCED_FLOW_CONTROL, this.advancedFlowControlEnabled);
setOutstandingHandshakeLimit(Integer.getInteger(OUTSTANDING_HANDSHAKE_LIMIT, -1));
//bossgroup处理accept事件
EventLoopGroup bossGroup = NettyUtils.newNioOrEpollEventLoopGroup(NettyUtils.getClientReachableLocalInetAddressCount());
//启动workgroup处理网络IO
EventLoopGroup workerGroup = NettyUtils.newNioOrEpollEventLoopGroup();
//配置引导分配器
ServerBootstrap bootstrap = new ServerBootstrap().group(bossGroup, workerGroup)
.channel(NettyUtils.nioOrEpollServerSocketChannel())
// 父信道选项
.option(ChannelOption.SO_REUSEADDR, true)
// 子信道选项
.childOption(ChannelOption.TCP_NODELAY, true)
.childOption(ChannelOption.SO_LINGER, -1)
.childHandler(new ChannelInitializer<SocketChannel>() {
@Override//初始化信道 protected void initChannel(SocketChannel ch) throws Exception {
ChannelPipeline pipeline = ch.pipeline();
if (advancedFlowControlEnabled) {
pipeline.addLast(readIssuedTrackingHandler);
}
if (secure) {
initSSL(pipeline, false);
} else if (shouldUsePortUnification) {
initSSL(pipeline, true);
}
pipeline.addLast("servercnxnfactory", channelHandler);
}
});
this.bootstrap = configureBootstrapAllocator(bootstrap);
this.bootstrap.validate();
}
总结
NettyServerCnxnFactory与NIONettyServerCnxnFactory之间的差异
| 差异 | NIOServerCnxnFactory | NettyServerCnxnFactory |
|---|---|---|
| accept事件 | 启动1个accept thread | boss group处理accept事件,默认启动1个线程 |
| select | 启动select thread | 调用addLast(EventExecutorGroup, ChannelHandler…) |
| 网络IO | 启动worker thread | 启动work group处理网络IO,默认启动核心数∗2个线程 |
| 处理读事件 | 在worker thread中调用NIOServerCnxn.doIO()处理 | 在handler中处理读事件 |
| 处理写事件 | 执行FinalRP.processRequest()的线程与worker thread通过NIOServerCnxn.outgoingBuffers进行通信,由worker thread批量写 | netty天生支持异步写,若当前线程为EventLoop线程,则将待写入数据存放到ChannelOutboundBuffer中.若当前线程不是EventLoop线程,构造写任务添加至EventLoop任务队列中 |