基于Zookeeper+Thrift的RPC动态服务注册发现和调用(Java)

一 、介绍一下使用到的框架类工具以及pom文件

(1) ZK封装非常好的框架类： Curator （可以先去学习）

(2)  Thrift文件编译工具， 本人安装的老版本 Thrift Compiler (0.9.3) 

(3) ZK UI（可以忽略）

(4) pom文件

            
                org.apache.curator
                curator-framework
                4.0.0
            
            
                org.apache.curator
                curator-recipes
                4.0.0
            
            
                org.apache.curator
                curator-x-discovery
                4.0.0
            
            
                org.apache.curator
                curator-test
                4.0.0
                test
            
            
                org.apache.thrift
                libthrift
                0.9.3
            

二 、相关代码和工程（代码内注释详解）

（1）ZK 客户端 服务类

package com.play.english.cqx.zk;

import org.apache.curator.RetryPolicy;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.framework.recipes.cache.PathChildrenCache;
import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener;
import org.apache.curator.retry.RetryForever;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.apache.curator.utils.CloseableUtils;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.data.ACL;

import java.util.List;

/**
 * @author chaiqx on 2019/12/3
 */
public class CqxZk implements AutoCloseable {

    //zk 服务名称
    private String name;

    //zk 服务器连接字符串
    private String zkConnectedStr;

    //封装好的client
    private CuratorFramework client;

    public String getName() {
        return name;
    }

    public void setName(String name) {
        this.name = name;
    }

    public String getZkConnectedStr() {
        return zkConnectedStr;
    }

    public void setZkConnectedStr(String zkConnectedStr) {
        this.zkConnectedStr = zkConnectedStr;
    }

    public CuratorFramework getClient() {
        return client;
    }

    public void setClient(CuratorFramework client) {
        this.client = client;
    }


    /**
     * 自定义一个异常捕获处理器,只是打印暂时无别的操作
     */
    private Thread.UncaughtExceptionHandler uncaughtExceptionHandler = new Thread.UncaughtExceptionHandler() {
        @Override
        public void uncaughtException(Thread t, Throwable e) {
            System.out.println(name + ": " + e);
        }
    };

    /**
     * 构造函数，启动zk客户端和连接zk服务器
     *
     * @param name
     * @param zkConnectedStr
     */
    public CqxZk(String name, String zkConnectedStr) {
        try {
            this.name = name;
            this.zkConnectedStr = zkConnectedStr;
            RetryPolicy retryPolicy = new RetryForever(10000);
            this.client = CuratorFrameworkFactory.builder()
                    .connectString(zkConnectedStr)
                    .retryPolicy(retryPolicy)
                    .sessionTimeoutMs(30 * 1000)
                    .connectionTimeoutMs(30 * 1000)
                    .maxCloseWaitMs(60 * 1000)
                    .threadFactory(new ThreadFactoryBuilder().setNameFormat(name + "-%d").setUncaughtExceptionHandler(uncaughtExceptionHandler).build())
                    .build();
            this.client.start();
            this.client.blockUntilConnected();
            System.out.println(String.format("cqx zk :  %s started.", this.zkConnectedStr));
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            throw new CqxZkException(e);
        }
    }

    /**
     * 添加节点
     *
     * @param path
     * @param value
     * @param mode
     * @return
     */
    public String add(String path, byte[] value, CreateMode mode) {
        try {
            return this.client.create().creatingParentsIfNeeded().withMode(mode).forPath(path, value);
        } catch (Exception e) {
            throw new CqxZkException(e);
        }
    }

    /**
     * 添加节点
     *
     * @param path
     * @param value
     * @param mode
     * @param aclList
     * @return
     */
    public String add(String path, byte[] value, CreateMode mode, List aclList) {
        try {
            return this.client.create().creatingParentsIfNeeded().withMode(mode).withACL(aclList).forPath(path, value);
        } catch (Exception e) {
            throw new CqxZkException(e);
        }
    }

    /**
     * 判断节点是否存在
     *
     * @param path
     * @return
     */
    public boolean exist(String path) {
        try {
            return this.client.checkExists().forPath(path) != null;
        } catch (Exception e) {
            System.out.println(String.format("zk check exist error, path = %s , %s", path, e));
            return false;
        }
    }

    /**
     * 移除节点
     *
     * @param path
     */
    public void remove(String path) {
        try {
            this.client.delete().forPath(path);
        } catch (Exception e) {
            throw new CqxZkException(e);
        }
    }

    /**
     * 设置节点数据
     *
     * @param path
     * @param value
     */
    public void set(String path, byte[] value) {
        try {
            this.client.setData().forPath(path, value);
        } catch (Exception e) {
            throw new CqxZkException(e);
        }
    }

    /**
     * 获取节点下的所有子节点
     *
     * @param nodePath
     * @return
     */
    public List getChildren(String nodePath) {
        try {
            return this.client.getChildren().forPath(nodePath);
        } catch (Exception e) {
            System.out.println(String.format("get node children failed, nodePath = %s ", nodePath));
        }
        return null;
    }

    /**
     * 注册目录监听器
     *
     * @param nodePath
     * @param listener
     * @return
     */
    public PathChildrenCache registerPathChildrenListener(String nodePath, PathChildrenCacheListener listener) {
        try {
            //创建一个PathChildrenCache
            PathChildrenCache pathChildrenCache = new PathChildrenCache(this.client, nodePath, true);
            //添加子目录监视器
            pathChildrenCache.getListenable().addListener(listener);
            //启动监听器
            pathChildrenCache.start(PathChildrenCache.StartMode.BUILD_INITIAL_CACHE);
            //返回PathChildrenCache
            return pathChildrenCache;
        } catch (Exception e) {
            System.out.println(String.format("register path children node cache listener failed, nodePath = %s ", nodePath));
        }
        return null;
    }

    @Override
    public void close() throws Exception {
        System.out.println(String.format("cqx zk %s - %s closed", this.name, this.zkConnectedStr));
        CloseableUtils.closeQuietly(this.client);
    }
}

(2) Thrift 源文件Hello.thrift（包名自己自定义）

namespace java com.play.english.cqx.thrift.thrift

service hello{
       string sayHello(1:i32 id)
}

编译脚本：(注意执行目录)

#!/usr/bin/env bash

thrift --gen java -out ../thrift Hello.thrift

执行脚本最终生成java代码类：Hello.java 

(3) Thrift 服务端 Server类

Thrift  RPC Service 具体实现类:

package com.play.english.cqx.thrift.server;

import com.play.english.cqx.thrift.thrift.Hello;
import org.apache.thrift.TException;

/**
 * @author chaiqx on 2019/12/9
 */
public class HelloImpl implements Hello.Iface {

    @Override
    public String sayHello(int id) throws TException {
        if (id == 1) {
            return "hello, I am cqx!";
        } else if (id == 2) {
            return "hello, I am cqh!";
        } else {
            return "hello";
        }
    }
}

Thrift PRC Server 服务类：

package com.play.english.cqx.thrift.server;

import com.play.english.cqx.thrift.thrift.Hello;
import com.play.english.cqx.zk.CqxZk;
import org.apache.commons.lang.StringUtils;
import org.apache.thrift.protocol.TBinaryProtocol;
import org.apache.thrift.server.TServer;
import org.apache.thrift.server.TThreadedSelectorServer;
import org.apache.thrift.transport.TFramedTransport;
import org.apache.thrift.transport.TNonblockingServerSocket;
import org.apache.thrift.transport.TTransportException;
import org.apache.zookeeper.CreateMode;

import java.net.InetAddress;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;

/**
 * @author chaiqx on 2019/12/9
 */
public class CqxThriftServer {

    //RPC所有相关节点公用的顶级父节点
    private static final String THRIFT_SERVER_PREFIX = "/thrift/";
    //固定的一个单线程池
    private ExecutorService thread = Executors.newSingleThreadExecutor();
    //zk客户端服务实例
    private CqxZk cqxZk;
    //RPC服务名
    private String name;
    //PRC服务端口号
    private int port;

    /**
     * 构造函数，启动RPC节点并注册节点到ZK
     *
     * @param port
     * @param name
     * @param cqxZk
     */
    public CqxThriftServer(int port, String name, CqxZk cqxZk) {
        this.cqxZk = cqxZk;
        this.name = name;
        this.port = port;
        this.startAndRegisterService();
    }

    /**
     * 获取本服务即将注册到ZK上的节点路径
     * 根据本机器IP和定义的端口号生成唯一路径
     *
     * @return
     */
    private String getServiceNodePath() {
        try {
            InetAddress inetAddress = InetAddress.getLocalHost();
            String servicePath = inetAddress.getHostAddress();
            return "/".concat(servicePath).concat(":").concat(String.valueOf(port));
        } catch (Exception e) {
            return null;
        }
    }

    /**
     * 注册RPC服务父节点
     * 比如我们此RPC服务为hello-server
     * 注册完之后就是/thrift/hello-server 永久节点
     */
    private void register() {
        if (!cqxZk.exist(THRIFT_SERVER_PREFIX.concat(name))) {
            cqxZk.add(THRIFT_SERVER_PREFIX.concat(name), name.getBytes(), CreateMode.PERSISTENT);
        }
    }

    /**
     * 注册此RPC服务节点
     * 

     * 注册完之后就是/thrift/hello-server/10.1.38.226:7778 临时节点
     */
    private void registerService() {
        if (!cqxZk.exist(THRIFT_SERVER_PREFIX.concat(name))) {
            register();
        }
        String serviceNodePath = getServiceNodePath();
        if (StringUtils.isBlank(serviceNodePath)) {
            return;
        }
        cqxZk.add(THRIFT_SERVER_PREFIX.concat(name).concat(serviceNodePath), String.valueOf(port).getBytes(), CreateMode.EPHEMERAL);
    }

    /**
     * 启动RPC服务
     *
     * @return
     */
    private boolean start() {
        try {
            //构造thrift-server
            TServer server = new TThreadedSelectorServer(new TThreadedSelectorServer.Args(new TNonblockingServerSocket(port))
                    .protocolFactory(new TBinaryProtocol.Factory())
                    .processor(new Hello.Processor(new HelloImpl()))
                    .workerThreads(5)
                    .transportFactory(new TFramedTransport.Factory()));
            //异步线程提交,防止主线程阻塞
            thread.submit(server::serve);
            //一直轮训等待RPC服务启动成功,服务正常
            while (!server.isServing()) {
                System.out.println("wait for thrift server start!");
                try {
                    TimeUnit.SECONDS.sleep(1);
                } catch (InterruptedException e) {
                    System.out.println(e);
                }
            }
        } catch (TTransportException e) {
            System.out.println(e);
            return false;
        }
        return true;
    }

    /**
     * 启动RPC服务并且注册ZK节点
     */
    private void startAndRegisterService() {
        if (!start()) {
            System.out.println(name.concat("start failed!"));
        }
        registerService();
    }


    public static void main(String[] args) {
        //创建ZK客户端实例
        CqxZk cqxZk = new CqxZk("test-thrift", "127.0.0.1:2181");
        //异步启动第一个RPC服务
        new Thread("hello-server-1") {
            @Override
            public void run() {
                new CqxThriftServer(7777, "hello-server", cqxZk);
            }
        }.start();
        //异步启动第二个RPC服务
        new Thread("hello-server-2") {
            @Override
            public void run() {
                new CqxThriftServer(7778, "hello-server", cqxZk);
            }
        }.start();
    }
}

(4) Thrift 客户端 Client类

package com.play.english.cqx.thrift.client;

import com.play.english.cqx.thrift.thrift.Hello;
import com.play.english.cqx.zk.CqxZk;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.curator.framework.recipes.cache.ChildData;
import org.apache.thrift.TException;
import org.apache.thrift.protocol.TBinaryProtocol;
import org.apache.thrift.protocol.TProtocol;
import org.apache.thrift.transport.TFramedTransport;
import org.apache.thrift.transport.TSocket;
import org.apache.thrift.transport.TTransport;
import org.apache.thrift.transport.TTransportException;

import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;

/**
 * @author chaiqx on 2019/12/9
 */
public class CqxThriftClient {

    //自己知道自己的PRC服务节点路径所以写死
    private static final String NODE_PATH = "/thrift/hello-server";

    //IP和端口号分隔符
    private static final String SPLIT_STR = ":";

    //ZK客户端实例
    private CqxZk cqxZk;

    //RPC服务连接池,Map本地简单存储
    private Map protocolMap = new ConcurrentHashMap<>();

    /**
     * 构造函数，RPC服务发现和RPC服务动态监听
     *
     * @param cqxZk
     */
    public CqxThriftClient(CqxZk cqxZk) {
        this.cqxZk = cqxZk;
        this.serverDetect();
        this.serverListening();
    }

    /**
     * 获取一个RPC服务连接
     *
     * @return
     */
    private TProtocol getProtocol() {
        //如果连接池为空，则返回空
        if (MapUtils.isEmpty(protocolMap)) {
            return null;
        }
        //随机取出一个RPC服务连接
        List tmp = new ArrayList<>(new ArrayList<>(protocolMap.values()));
        Collections.shuffle(tmp);
        //打印一下使用的那个连接
        System.out.println(tmp.get(0).toString());
        return tmp.get(0);
    }

    /**
     * 对应zk节点路径的RPC服务进连接池
     *
     * @param path
     */
    private void inProtocolPool(String path) {
        if (StringUtils.isBlank(path)) {
            return;
        }
        //解析该节点的服务地址
        String[] address = path.split(SPLIT_STR);
        if (ArrayUtils.isEmpty(address) || address.length != 2) {
            return;
        }
        if (protocolMap.containsKey(path)) {
            return;
        }
        try {
            //创建与PRC服务端的连接
            TTransport tTransport = new TSocket(address[0], Integer.parseInt(address[1]));
            TFramedTransport framedTransport = new TFramedTransport(tTransport);
            TProtocol tProtocol = new TBinaryProtocol(framedTransport);
            tTransport.open();
            //进连接池
            protocolMap.put(path, tProtocol);
        } catch (TTransportException e) {
            System.out.println(e);
        }
    }

    /**
     * 对应zk节点路径的RPC服务出连接池
     *
     * @param path
     */
    private void outProtocolPool(String path) {
        if (!protocolMap.containsKey(path)) {
            return;
        }
        //出连接池
        TProtocol protocol = protocolMap.remove(path);
        //关闭连接
        protocol.getTransport().close();
    }

    /**
     * 发现RPC服务
     */
    private void serverDetect() {
        List childrenNodePaths = cqxZk.getChildren(NODE_PATH);
        if (childrenNodePaths != null) {
            childrenNodePaths.forEach(this::inProtocolPool);
        }
    }

    /**
     * 动态监听RPC服务
     */
    private void serverListening() {
        //注册ZK目录监听器
        cqxZk.registerPathChildrenListener(NODE_PATH, (curatorClient, event) -> {
            //获取变化的节点数据
            ChildData childData = event.getData();
            if (childData == null) {
                return;
            }
            switch (event.getType()) {
                case CHILD_ADDED://新增RPC节点
                    System.out.println(String.format("path children add children node %s now", childData.getPath()));
                    //新节点进RPC服务连接池
                    inProtocolPool(childData.getPath().substring(childData.getPath().lastIndexOf("/") + 1));
                    break;
                case CHILD_REMOVED://减少RPC节点
                    System.out.println(String.format("path children delete children node %s now", childData.getPath()));
                    //失去的节点出RPC服务连接池
                    outProtocolPool(childData.getPath().substring(childData.getPath().lastIndexOf("/") + 1));
                    break;
                case CONNECTION_LOST://RPC节点连接丢失
                    System.out.println(String.format("path children connection lost %s now", childData.getPath()));
                    //断开连接节点出RPC服务连接池
                    outProtocolPool(childData.getPath().substring(childData.getPath().lastIndexOf("/") + 1));
                    break;
                case CONNECTION_RECONNECTED://RPC节点重连
                    System.out.println(String.format("path children connection reconnected %s now", childData.getPath()));
                    //重新连接的节点出RPC服务连接池
                    inProtocolPool(childData.getPath().substring(childData.getPath().lastIndexOf("/") + 1));
                    break;
                default://无操作
                    break;
            }
        });
    }

    /**
     * 客户端say hello
     *
     * @param id
     * @return
     */
    private String sayHello(int id) {
        //获取一个RPC服务连接
        TProtocol protocol = this.getProtocol();
        if (protocol == null) {
            return null;
        }
        //创建一个RPC实例
        Hello.Client client = new Hello.Client(protocol);
        try {
            //RPC实际say hello
            return client.sayHello(id);
        } catch (TException e) {
            System.out.println(e);
        }
        return null;
    }

    public static void main(String[] args) throws Exception {
        //ZK客户端实例
        CqxZk cqxZk = new CqxZk("test-thrift", "127.0.0.1:2181");
        //Thrift 客户端
        CqxThriftClient cqxThriftClient = new CqxThriftClient(cqxZk);
        //每五秒就打印三次say hello,结果可想而知，使用的rpc服务不是同一个，会随机选取调用
        while (true) {
            for (int i = 0; i < 3; i++) {
                System.out.println(cqxThriftClient.sayHello(i));
            }
            TimeUnit.SECONDS.sleep(5);
        }
    }
}

(5)结果以及结论

如果先启动RPC客户端，再启动RPC服务端：

 刚开始的时候没有RPC服务，所以一直say hello 提示是null

然后RPC服务端启动之后，RPC客户端监听到ZK节点变化，然后获取RPC并连接上RPC服务，然后调用say hello 就有数据了，

而且是两个RPC节点，所以会不断的随机选取一个服务。（如果先启动RPC服务端然后再启动RPC客户端呢？）