dubbo源码分析之--java.lang.IllegalStateException: urls to invokers error .invokerUrls.size :1, invoker.si...

2019独角兽企业重金招聘Python工程师标准>>>

最近在生产环境和预发环境经常出现java.lang.IllegalStateException: urls to invokers error .invokerUrls.size :1, invoker.size :0，这一类的error错误，具体错误信息如下

java.lang.IllegalStateException: urls to invokers error .invokerUrls.size :1, invoker.size :0. urls :[dubbo://192.168.137.13:20881/com.alibaba.dubbo.demo.DemoService?anyhost=true&application=dubbodemoprovider&dubbo=2.5.3.6&interface=com.alibaba.dubbo.demo.DemoService&methods=complexInvoke,complex,Timeout,complexObject,complexUser,sayHello2,$$sayHelloTest,complexArray2,hessianTest,sayHello,HelloWorld,hessianMapTest,complexArray,hessianSerialable,ThrowException,mockDate&pid=3094&revision=2.5.3.8-SNAPSHOT&side=provider×tamp=1493780202580]
at com.alibaba.dubbo.registry.integration.RegistryDirectory.refreshInvoker(RegistryDirectory.java:229) [dubbo-2.5.3.9.jar:2.5.3.9]
    at com.alibaba.dubbo.registry.integration.RegistryDirectory.notify(RegistryDirectory.java:195) [dubbo-2.5.3.9.jar:2.5.3.9]
   at com.alibaba.dubbo.registry.support.AbstractRegistry.notify(AbstractRegistry.java:449) [dubbo-2.5.3.9.jar:2.5.3.9]
at com.alibaba.dubbo.registry.support.FailbackRegistry.doNotify(FailbackRegistry.java:273) [dubbo-2.5.3.9.jar:2.5.3.9]
    at com.alibaba.dubbo.registry.support.FailbackRegistry.notify(FailbackRegistry.java:259) [dubbo-2.5.3.9.jar:2.5.3.9]

at com.alibaba.dubbo.registry.zookeeper.ZookeeperRegistry.access$400(ZookeeperRegistry.java:45) [dubbo-2.5.3.9.jar:2.5.3.9]
at com.alibaba.dubbo.registry.zookeeper.ZookeeperRegistry$3.childChanged(ZookeeperRegistry.java:163) [dubbo-2.5.3.9.jar:2.5.3.9]
at com.alibaba.dubbo.remoting.zookeeper.zkclient.ZkclientZookeeperClient$2.handleChildChange(ZkclientZookeeperClient.java:82) [dubbo-2.5.3.9.jar:2.5.3.9]
    at org.I0Itec.zkclient.ZkClient$7.run(ZkClient.java:568) [zkclient-0.1.jar:na]
    at org.I0Itec.zkclient.ZkEventThread.run(ZkEventThread.java:71) [zkclient-0.1.jar:na]

首先出现该类型错误，并不影响正常调用，dubbo中有保护机制，当所有的的provider都被disable后，会保留最后一个被disable的provider，提供正常的服务。具体的实现代码如下RegistryDirectory类中的如下方法

private void refreshInvoker(List invokerUrls){
    if (invokerUrls != null && invokerUrls.size() == 1 && invokerUrls.get(0) != null
            && Constants.EMPTY_PROTOCOL.equals(invokerUrls.get(0).getProtocol())) {
        this.forbidden = true; // 禁止访问
        this.methodInvokerMap = null; // 置空列表
        destroyAllInvokers(); // 关闭所有Invoker
    } else {
        this.forbidden = false; // 允许访问
        Map> oldUrlInvokerMap = this.urlInvokerMap; // local reference
        if (invokerUrls.size() == 0 && this.cachedInvokerUrls != null){
            invokerUrls.addAll(this.cachedInvokerUrls);
        } else {
            this.cachedInvokerUrls = new HashSet();
            this.cachedInvokerUrls.addAll(invokerUrls);//缓存invokerUrls列表，便于交叉对比
        }
        if (invokerUrls.size() ==0 ){
           return;
        }
        Map> newUrlInvokerMap = toInvokers(invokerUrls) ;// 将URL列表转成Invoker列表
        Map>> newMethodInvokerMap = toMethodInvokers(newUrlInvokerMap); // 换方法名映射Invoker列表
        // state change
        //如果计算错误，则不进行处理.
        //这里是抛出具体错误的地方，当newUrlInvokerMap都为空时，这里会打印出ERROR日志
        if (newUrlInvokerMap == null || newUrlInvokerMap.size() == 0 ){
            logger.error(new IllegalStateException("urls to invokers error .invokerUrls.size :"+invokerUrls.size() + ", invoker.size :0. urls :"+invokerUrls.toString()));
            return ;
        }
        this.methodInvokerMap = multiGroup ? toMergeMethodInvokerMap(newMethodInvokerMap) : newMethodInvokerMap;
        this.urlInvokerMap = newUrlInvokerMap;
        try{
            destroyUnusedInvokers(oldUrlInvokerMap,newUrlInvokerMap); // 关闭未使用的Invoker
        }catch (Exception e) {
            logger.warn("destroyUnusedInvokers error. ", e);
        }
    }
}

具体是在进行灰度发布时，会先进行disable，会通过zookeeper通知到客户端，进行连接销毁，因为disable操作是单个进行，所以当操作到最后一个时，如果发现没有可用提供者，则会打印error日志，保留最后一个如下是AbstractRegistry类中的通知

protected void notify(URL url, NotifyListener listener, List urls) {
    if (url == null) {
        throw new IllegalArgumentException("notify url == null");
    }
    if (listener == null) {
        throw new IllegalArgumentException("notify listener == null");
    }
    if ((urls == null || urls.size() == 0) 
            && ! Constants.ANY_VALUE.equals(url.getServiceInterface())) {
        logger.warn("Ignore empty notify urls for subscribe url " + url);
        return;
    }
    if (logger.isDebugEnabled()) {
        logger.debug("Notify urls for subscribe url " + url + ", urls: " + urls);
    }
    Map> result = new HashMap>();
    for (URL u : urls) {
        if (UrlUtils.isMatch(url, u)) {
           String category = u.getParameter(Constants.CATEGORY_KEY, Constants.DEFAULT_CATEGORY);
           List categoryList = result.get(category);
           if (categoryList == null) {
              categoryList = new ArrayList();
              result.put(category, categoryList);
           }
           categoryList.add(u);
        }
    }
    if (result.size() == 0) {
        return;
    }
    Map> categoryNotified = notified.get(url);
    if (categoryNotified == null) {
        notified.putIfAbsent(url, new ConcurrentHashMap>());
        categoryNotified = notified.get(url);
    }
    for (Map.Entry> entry : result.entrySet()) {
        String category = entry.getKey();
        List categoryList = entry.getValue();
        categoryNotified.put(category, categoryList);
        saveProperties(url);
        listener.notify(categoryList);
    }
}

RegistryDirectory类中的通知

public synchronized void notify(List urls) {
    List invokerUrls = new ArrayList();
    List routerUrls = new ArrayList();
    List configuratorUrls = new ArrayList();
    for (URL url : urls) {
        String protocol = url.getProtocol();
        String category = url.getParameter(Constants.CATEGORY_KEY, Constants.DEFAULT_CATEGORY);
        if (Constants.ROUTERS_CATEGORY.equals(category) 
                || Constants.ROUTE_PROTOCOL.equals(protocol)) {
            routerUrls.add(url);
        } else if (Constants.CONFIGURATORS_CATEGORY.equals(category) 
                || Constants.OVERRIDE_PROTOCOL.equals(protocol)) {
            configuratorUrls.add(url);
        } else if (Constants.PROVIDERS_CATEGORY.equals(category)) {
            invokerUrls.add(url);
        } else {
            logger.warn("Unsupported category " + category + " in notified url: " + url + " from registry " + getUrl().getAddress() + " to consumer " + NetUtils.getLocalHost());
        }
    }
    // configurators 
    if (configuratorUrls != null && configuratorUrls.size() >0 ){
        this.configurators = toConfigurators(configuratorUrls);
    }
    // routers
    if (routerUrls != null && routerUrls.size() >0 ){
        List routers = toRouters(routerUrls);
        if(routers != null){ // null - do nothing
            setRouters(routers);
        }
    }
    List localConfigurators = this.configurators; // local reference
    // 合并override参数
    this.overrideDirectoryUrl = directoryUrl;
    if (localConfigurators != null && localConfigurators.size() > 0) {
        for (Configurator configurator : localConfigurators) {
            this.overrideDirectoryUrl = configurator.configure(overrideDirectoryUrl);
        }
    }
    // providers
    refreshInvoker(invokerUrls);
}

下面的连接销毁的方法是DubboInvoker中

//destroyUnusedInvokers这个方法中真正去销毁时会去调用该方法
public void destroy() {
        //防止client被关闭多次.在connect per jvm的情况下，client.close方法会调用计数器-1，当计数器小于等于0的情况下，才真正关闭
        if (super.isDestroyed()){
        return ;
    } else {
        //dubbo check ,避免多次关闭
        destroyLock.lock();
        try{
            if (super.isDestroyed()){
                return ;
            }
            super.destroy();
            if (invokers != null){
                invokers.remove(this);
            }
            for (ExchangeClient client : clients) {
                try {
                    client.close();
                } catch (Throwable t) {
                    logger.warn(t.getMessage(), t);
                }
            }
            
        }finally {
            destroyLock.unlock();
        }
    }
}

转载于:https://my.oschina.net/u/1262062/blog/891439