nats源码阅读记录-消息广播

记录个人理解，如果有错误请给与指正，谢谢；

依照消息订阅 发布模式，阅读相关代码

消息在一个ROUTER中的扩算

消息在一个集群内多个ROUTER之间的扩算

消息跨集群扩算

消息从publisher发布出去后，进入到队列此处对应的client.然后扩算到对应的订阅者（sub)列表。

func (c *client) readLoop() {

//监听读，读到数据后，进入PARSE函数进行数据解析
   for {
      n, err := nc.Read(b)
      if err != nil {
         if err == io.EOF {
            c.closeConnection(ClientClosed)
         } else {
            c.closeConnection(ReadError)
         }
         return
      }

      // Clear inbound stats cache
      c.in.msgs = 0
      c.in.bytes = 0
      c.in.subs = 0

      // Main call into parser for inbound data. This will generate callouts
      // to process messages, etc.
      if err := c.parse(b[:n]); err != nil {
         // handled inline
         if err != ErrMaxPayload && err != ErrAuthentication {
            c.Errorf("%s", err.Error())
            c.closeConnection(ProtocolViolation)
         }
         return
      }

      // Updates stats for client and server that were collected
      // from parsing through the buffer.
      if c.in.msgs > 0 {
         atomic.AddInt64(&c.inMsgs, int64(c.in.msgs))
         atomic.AddInt64(&c.inBytes, int64(c.in.bytes))
         atomic.AddInt64(&s.inMsgs, int64(c.in.msgs))
         atomic.AddInt64(&s.inBytes, int64(c.in.bytes))
      }

      // Budget to spend in place flushing outbound data.
      // Client will be checked on several fronts to see
      // if applicable. Routes and Gateways will never
      // spend time flushing outbound in place.
      var budget time.Duration
      if c.kind == CLIENT {
         budget = time.Millisecond
      }

//收到数据，并且处理结束后，通知写协程把相关信息给发送出(接收信号在writelooop()中)
      // Flush, or signal to writeLoop to flush to socket.
      c.flushClients(budget)

      // Update activity, check read buffer size.
      c.mu.Lock()
      nc := c.nc

      // Activity based on interest changes or data/msgs.
      if c.in.msgs > 0 || c.in.subs > 0 {
         c.last = time.Now()
      }

      if n >= cap(b) {
         c.in.srs = 0
      } else if n < cap(b)/2 { // divide by 2 b/c we want less than what we would shrink to.
         c.in.srs++
      }
      //根据消息大小，重新调整所需要的内存空间
      // Update read buffer size as/if needed.
      if n >= cap(b) && cap(b) < maxBufSize {
         // Grow
         c.in.rsz = int32(cap(b) * 2)
         b = make([]byte, c.in.rsz)
      } else if n < cap(b) && cap(b) > minBufSize && c.in.srs > shortsToShrink {
         // Shrink, for now don't accelerate, ping/pong will eventually sort it out.
         c.in.rsz = int32(cap(b) / 2)
         b = make([]byte, c.in.rsz)
      }
      c.mu.Unlock()

      // Check to see if we got closed, e.g. slow consumer
      if nc == nil {
         return
      }
   }
}

parse函数较长，主要是从收到的信息里面解析出来几个请求，参考一下剩余代码 

func (c *client) parse(buf []byte) error {
 

   // Move to loop instead of range syntax to allow jumping of i
   for i = 0; i < len(buf); i++ {
      b = buf[i]

      switch c.state {
      case OP_START:
         
      case PUB_ARG:
         switch b {
         case '\r':
            c.drop = 1
         case '\n':
            var arg []byte
            if c.argBuf != nil {
               arg = c.argBuf
               c.argBuf = nil
            } else {
               arg = buf[c.as : i-c.drop]
            }
            //解析出来发布信息
            if err := c.processPub(c.trace, arg); err != nil {
               return err
            }
            c.drop, c.as, c.state = OP_START, i+1, MSG_PAYLOAD
            // If we don't have a saved buffer then jump ahead with
            // the index. If this overruns what is left we fall out
            // and process split buffer.
            if c.msgBuf == nil {
               i = c.as + c.pa.size - LEN_CR_LF
            }
         default:
            if c.argBuf != nil {
               c.argBuf = append(c.argBuf, b)
            }
         }
      case MSG_PAYLOAD:   
      case MSG_END_R:
      case MSG_END_N:
         if b != '\n' {
            goto parseErr
         }
         if c.msgBuf != nil {
            c.msgBuf = append(c.msgBuf, b)
         } else {
            c.msgBuf = buf[c.as : i+1]
         }
        解析出来要扩算的消息
         c.processInboundMsg(c.msgBuf)
         fmt.Println("*********", string(c.msgBuf))
         c.argBuf, c.msgBuf = nil, nil
         c.drop, c.as, c.state = 0, i+1, OP_START
         // Drop all pub args
         c.pa.arg, c.pa.pacache, c.pa.account, c.pa.subject = nil, nil, nil, nil
         c.pa.reply, c.pa.szb, c.pa.queues = nil, nil, nil
      case OP_A:
     
      case OP_ASUB:     
      case OP_ASUB_SPC:       
      case ASUB_ARG:
            // 跨集群的订阅
            if err := c.processAccountSub(arg); err != nil {
               return err
            }
            c.drop, c.as, c.state = 0, i+1, OP_START
         default:
            if c.argBuf != nil {
               c.argBuf = append(c.argBuf, b)
            }
         }
     
      case AUSUB_ARG:
         switch b {
         case '\r':
            c.drop = 1
         case '\n':
            var arg []byte
            if c.argBuf != nil {
               arg = c.argBuf
               c.argBuf = nil
            } else {
               arg = buf[c.as : i-c.drop]
            }
           取消跨集群订阅
            c.processAccountUnsub(arg)
            c.drop, c.as, c.state = 0, i+1, OP_START
         }
     
      case SUB_ARG:
         switch b {
         case '\r':
            c.drop = 1
         case '\n':
            var arg []byte
            if c.argBuf != nil {
               arg = c.argBuf
               c.argBuf = nil
            } else {
               arg = buf[c.as : i-c.drop]
            }
            var err error

            switch c.kind {
            case CLIENT:
               本router内的订阅（订阅者和router直接链接通信）
               err = c.processSub(arg)
            case ROUTER:
               别的router收到的订阅信息，扩算到本router中
               err = c.processRemoteSub(arg)
            case GATEWAY:
               别的集群信息扩算到本集群
               err = c.processGatewayRSub(arg)
            }
            if err != nil {
               return err
            }
            c.drop, c.as, c.state = 0, i+1, OP_START
         default:
            if c.argBuf != nil {
               c.argBuf = append(c.argBuf, b)
            }
         }
      
      case UNSUB_ARG:
         switch b {
         case '\r':
            c.drop = 1
         case '\n':
            var arg []byte
            if c.argBuf != nil {
               arg = c.argBuf
               c.argBuf = nil
            } else {
               arg = buf[c.as : i-c.drop]
            }
            var err error
            取消所有订阅
            switch c.kind {
            case CLIENT:
               err = c.processUnsub(arg)
            case ROUTER:
               err = c.processRemoteUnsub(arg)
            case GATEWAY:
               err = c.processGatewayRUnsub(arg)
            }
            if err != nil {
               return err
            }
            c.drop, c.as, c.state = 0, i+1, OP_START
         default:
            if c.argBuf != nil {
               c.argBuf = append(c.argBuf, b)
            }
         }
               
      case CONNECT_ARG:
         switch b {
         case '\r':
            c.drop = 1
         case '\n':
            var arg []byte
            if c.argBuf != nil {
               arg = c.argBuf
               c.argBuf = nil
            } else {
               arg = buf[c.as : i-c.drop]
            } 
           普通消息订阅/构建集群的跨router/构建多集群跨gateway请求（构建链接）
            if err := c.processConnect(arg); err != nil {
               return err
            }
            c.drop, c.state = 0, OP_START
            // Reset notion on authSet
            c.mu.Lock()
            authSet = c.awaitingAuth()
            c.mu.Unlock()
         default:
            if c.argBuf != nil {
               c.argBuf = append(c.argBuf, b)
            }
         }
      
      case MSG_ARG:
         switch b {
         case '\r':
            c.drop = 1
         case '\n':
            var arg []byte
            if c.argBuf != nil {
               arg = c.argBuf
               c.argBuf = nil
            } else {
               arg = buf[c.as : i-c.drop]
            }
            接收到 另外一个ROUTER扩散到来的发布消息
            if err := c.processRoutedMsgArgs(c.trace, arg); err != nil {
               return err
            }
            c.drop, c.as, c.state = 0, i+1, MSG_PAYLOAD

            // jump ahead with the index. If this overruns
            // what is left we fall out and process split
            // buffer.
            i = c.as + c.pa.size - LEN_CR_LF
         default:
            if c.argBuf != nil {
               c.argBuf = append(c.argBuf, b)
            }
         }
      
      case INFO_ARG:
         switch b {
         case '\r':
            c.drop = 1
         case '\n':
            var arg []byte
            if c.argBuf != nil {
               arg = c.argBuf
               c.argBuf = nil
            } else {
               arg = buf[c.as : i-c.drop]
            }
            链接建立后，协议握手
            if err := c.processInfo(arg); err != nil {
               return err
            }
            c.drop, c.as, c.state = 0, i+1, OP_START
         default:
            if c.argBuf != nil {
               c.argBuf = append(c.argBuf, b)
            }
         }
   }

   // Check for split buffer scenarios for any ARG state.
   if c.state == SUB_ARG || c.state == UNSUB_ARG || c.state == PUB_ARG ||
      c.state == ASUB_ARG || c.state == AUSUB_ARG ||
      c.state == MSG_ARG || c.state == MINUS_ERR_ARG ||
      c.state == CONNECT_ARG || c.state == INFO_ARG {
      // Setup a holder buffer to deal with split buffer scenario.
      if c.argBuf == nil {
         c.argBuf = c.scratch[:0]
         c.argBuf = append(c.argBuf, buf[c.as:i-c.drop]...)
      }
      // Check for violations of control line length here. Note that this is not
      // exact at all but the performance hit is too great to be precise, and
      // catching here should prevent memory exhaustion attacks.
      if len(c.argBuf) > int(mcl) {
         c.sendErr("Maximum Control Line Exceeded")
         c.closeConnection(MaxControlLineExceeded)
         return ErrMaxControlLine
      }
   }
}

消息来到本地，解析部分到此。作者根据字符一个一个的解析出消息，感觉这个方式可以改变。

消息本地扩散：

根据管理链接的的类型，走各自扩算消息策略

func (c *client) processInboundMsg(msg []byte) {
   switch c.kind {
   case CLIENT:
      c.processInboundClientMsg(msg)
   case ROUTER:
      c.processInboundRoutedMsg(msg)
   case GATEWAY:
      c.processInboundGatewayMsg(msg)
   }
}

func (c *client) processInboundClientMsg(msg []byte) {

 
   if c.opts.Verbose {
      收到信息后，给发送端一个回复
      c.sendOK()
   }

   // Mostly under testing scenarios.
   if c.srv == nil || c.acc == nil {
      return
   }

   // Match the subscriptions. We will use our own L1 map if
   // it's still valid, avoiding contention on the shared sublist.
   var r *SublistResult
   var ok bool

   genid := atomic.LoadUint64(&c.acc.sl.genid)
   if genid == c.in.genid && c.in.results != nil {
      r, ok = c.in.results[string(c.pa.subject)]
   } else {
      // Reset our L1 completely.
      c.in.results = make(map[string]*SublistResult)
      c.in.genid = genid
   }

获取对此消息感兴趣的订阅列表
   // Go back to the sublist data structure.
   if !ok {
      r = c.acc.sl.Match(string(c.pa.subject))
      c.in.results[string(c.pa.subject)] = r
      // Prune the results cache. Keeps us from unbounded growth. Random delete.
      if len(c.in.results) > maxResultCacheSize {
         n := 0
         for subject := range c.in.results {
            delete(c.in.results, subject)
            if n++; n > pruneSize {
               break
            }
         }
      }
   }

   // Check to see if we need to map/route to another account.
   if c.acc.imports.services != nil {
      c.checkForImportServices(c.acc, msg)
   }

   var qnames [][]byte

   // Check for no interest, short circuit if so.
   // This is the fanout scale.
   if len(r.psubs)+len(r.qsubs) > 0 {
      var collect bool
      // If we have queue subs in this cluster, then if we run in gateway
      // mode and the remote gateways have queue subs, then we need to
      // collect the queue groups this message was sent to so that we
      // exclude them when sending to gateways.
      if len(r.qsubs) > 0 && c.srv.gateway.enabled &&
         atomic.LoadInt64(&c.srv.gateway.totalQSubs) > 0 {
         collect = true
      }
     扩算消息函数，后面接着看此函数
      qnames = c.processMsgResults(c.acc, r, msg, c.pa.subject, c.pa.reply, collect)
   }

   // Now deal with gateways
   if c.srv.gateway.enabled { // 本地消息，如果配置了GATAWAY，那么消息同步给另外一个集群
      c.sendMsgToGateways(c.acc, msg, c.pa.subject, c.pa.reply, qnames)
   }
}

func (c *client) processMsgResults(acc *Account, r *SublistResult, msg, subject, reply []byte, collect bool) [][]byte {
   var queues [][]byte
   // msg header for clients.
   msgh := c.msgb[1:msgHeadProtoLen]
   msgh = append(msgh, subject...)
   msgh = append(msgh, ' ')
   si := len(msgh)

   // For sending messages across routes. Reset it if we have one.
   // We reuse this data structure.
   if c.in.rts != nil {
      c.in.rts = c.in.rts[:0]
   }

   每个订阅者都需要此消息
   for _, sub := range r.psubs {
      // Check if this is a send to a ROUTER. We now process
      // these after everything else.
      if sub.client.kind == ROUTER {
         if c.kind == ROUTER {
            //自己是ROUTER，又收到了别的ROUTER同步过来的MSG的时，消息不会再发给其他的router
            //消息不能无限反复扩散
            continue
         }
         c.addSubToRouteTargets(sub)
         continue
      } else if sub.client.kind == GATEWAY {
         // Never send to gateway from here.
         continue
      }
      // Check for stream import mapped subs. These apply to local subs only.
      if sub.im != nil && sub.im.prefix != "" {
         // Redo the subject here on the fly.
         msgh = c.msgb[1:msgHeadProtoLen]
         msgh = append(msgh, sub.im.prefix...)
         msgh = append(msgh, subject...)
         msgh = append(msgh, ' ')
         si = len(msgh)
      }
      // Normal delivery
      mh := c.msgHeader(msgh[:si], sub, reply)
      c.deliverMsg(sub, mh, msg)
   }

   // If we are sourced from a route we need to have direct filtered queues.
   //本客户端是路由，说明消息是别的ROUTER发来的。处理到此处结束，不再扩散消息
   if c.kind == ROUTER && c.pa.queues == nil {
      return queues
   }

   // Set these up to optionally filter based on the queue lists.
   // This is for messages received from routes which will have directed
   // guidance on which queue groups we should deliver to.
   qf := c.pa.queues

   // For gateway connections, we still want to send messages to routes
   // even if there are no queue filters.
   //消息从另外一个集群扩算到本集群，则立即进行本集群内的路由扩散
   if c.kind == GATEWAY && qf == nil {
      goto sendToRoutes
   }

   // Check to see if we have our own rand yet. Global rand
   // has contention with lots of clients, etc.
   if c.in.prand == nil {
      c.in.prand = rand.New(rand.NewSource(time.Now().UnixNano()))
   }

   // Process queue subs
   以下是摘自其他网站的注释。
   //排队Queueding 这个术语在不同的上下文有不同的意思，必须仔细区分其用法。
   // NATS实现了不支持持久化的分布式队列——通过订阅者的队列组（Queue Group）实现的。
   // 订阅者队列组提供了消息交付形式的负载均衡，主题(subject) 订阅既可以是“个体”订阅，又可以是队列组订阅。
   // 在创建订阅时，选择加入一个队列组，通过提供一个可选的队列组名。对于个体的 主题(subject) 订阅，
   // gnatsd服务会尝试交付该 主题(subject) 后续的每一条消息副本给每一个订阅者。
   // 而对于队列组的订阅，gnatsd 会尝试交付该 主题(subject) 后续的每一条消息给组中的任意一个订阅者，
   // 这个选择是随机的。分布式队列的这种交付形式是实时执行的，消息是不会持久化到二级存储中。
   // 此外，交付是基于 兴趣图（interest graph）—— 即订阅，所以它不是发布者的操作，而是完全受 gnatsd 的控制。
   for i := 0; i < len(r.qsubs); i++ {
      qsubs := r.qsubs[i]
      // If we have a filter check that here. We could make this a map or someting more
      // complex but linear search since we expect queues to be small. Should be faster
      // and more cache friendly.
      if qf != nil && len(qsubs) > 0 {
         tqn := qsubs[0].queue
         for _, qn := range qf {
            if bytes.Equal(qn, tqn) {
               消息发送给队列中的一个监听者。
               goto selectQSub
            }
         }
         continue
      }

   selectQSub:

      // We will hold onto remote qsubs when we are coming from a route
      // just in case we can no longer do local delivery.
      var rsub *subscription

      // Find a subscription that is able to deliver this message
      // starting at a random index.
      startIndex := c.in.prand.Intn(len(qsubs))
      for i := 0; i < len(qsubs); i++ {
         index := (startIndex + i) % len(qsubs)
         sub := qsubs[index]
         if sub == nil {
            continue
         }
         // Potentially sending to a remote sub across a route.
         if sub.client.kind == ROUTER {
            if c.kind == ROUTER {
               // We just came from a route, so skip and prefer local subs.
               // Keep our first rsub in case all else fails.
               if rsub == nil {
                  rsub = sub
               }
               continue
            } else {
               c.addSubToRouteTargets(sub)
               if collect {
                  queues = append(queues, sub.queue)
               }
            }
            break
         }
         // Check for mapped subs
         if sub.im != nil && sub.im.prefix != "" {
            // Redo the subject here on the fly.
            msgh = c.msgb[1:msgHeadProtoLen]
            msgh = append(msgh, sub.im.prefix...)
            msgh = append(msgh, subject...)
            msgh = append(msgh, ' ')
            si = len(msgh)
         }

         mh := c.msgHeader(msgh[:si], sub, reply)
         if c.deliverMsg(sub, mh, msg) {
            // Clear rsub
            rsub = nil
            if collect {
               queues = append(queues, sub.queue)
            }
            break
         }
      }

      if rsub != nil {
         // If we are here we tried to deliver to a local qsub
         // but failed. So we will send it to a remote.
         c.addSubToRouteTargets(rsub)
         if collect {
            queues = append(queues, rsub.queue)
         }
      }
   }
//一个集群内，路由间扩算
sendToRoutes:

   // If no messages for routes return here.
   if len(c.in.rts) == 0 {
      return queues
   }

   // We address by index to avoid struct copy.
   // We have inline structs for memory layout and cache coherency.
   for i := range c.in.rts {
      rt := &c.in.rts[i]

      mh := c.msgb[:msgHeadProtoLen]
      mh = append(mh, acc.Name...)
      mh = append(mh, ' ')
      mh = append(mh, subject...)
      mh = append(mh, ' ')

      if len(rt.qs) > 0 {
         if reply != nil {
            mh = append(mh, "+ "...) // Signal that there is a reply.
            mh = append(mh, reply...)
            mh = append(mh, ' ')
         } else {
            mh = append(mh, "| "...) // Only queues
         }
         mh = append(mh, rt.qs...)
      } else if reply != nil {
         mh = append(mh, reply...)
         mh = append(mh, ' ')
      }
      mh = append(mh, c.pa.szb...)
      mh = append(mh, _CRLF_...)
      c.deliverMsg(rt.sub, mh, msg)
   }
   return queues
}

参考：https://www.linuxidc.com/Linux/2016-10/136418.htm