记录个人理解,如果有错误请给与指正,谢谢;
依照消息订阅 发布模式,阅读相关代码
消息在一个ROUTER中的扩算
消息在一个集群内多个ROUTER之间的扩算
消息跨集群扩算
消息从publisher发布出去后,进入到队列此处对应的client.然后扩算到对应的订阅者(sub)列表。
func (c *client) readLoop() {
//监听读,读到数据后,进入PARSE函数进行数据解析
for {
n, err := nc.Read(b)
if err != nil {
if err == io.EOF {
c.closeConnection(ClientClosed)
} else {
c.closeConnection(ReadError)
}
return
}
// Clear inbound stats cache
c.in.msgs = 0
c.in.bytes = 0
c.in.subs = 0
// Main call into parser for inbound data. This will generate callouts
// to process messages, etc.
if err := c.parse(b[:n]); err != nil {
// handled inline
if err != ErrMaxPayload && err != ErrAuthentication {
c.Errorf("%s", err.Error())
c.closeConnection(ProtocolViolation)
}
return
}
// Updates stats for client and server that were collected
// from parsing through the buffer.
if c.in.msgs > 0 {
atomic.AddInt64(&c.inMsgs, int64(c.in.msgs))
atomic.AddInt64(&c.inBytes, int64(c.in.bytes))
atomic.AddInt64(&s.inMsgs, int64(c.in.msgs))
atomic.AddInt64(&s.inBytes, int64(c.in.bytes))
}
// Budget to spend in place flushing outbound data.
// Client will be checked on several fronts to see
// if applicable. Routes and Gateways will never
// spend time flushing outbound in place.
var budget time.Duration
if c.kind == CLIENT {
budget = time.Millisecond
}
//收到数据,并且处理结束后,通知写协程把相关信息给发送出(接收信号在writelooop()中)
// Flush, or signal to writeLoop to flush to socket.
c.flushClients(budget)
// Update activity, check read buffer size.
c.mu.Lock()
nc := c.nc
// Activity based on interest changes or data/msgs.
if c.in.msgs > 0 || c.in.subs > 0 {
c.last = time.Now()
}
if n >= cap(b) {
c.in.srs = 0
} else if n < cap(b)/2 { // divide by 2 b/c we want less than what we would shrink to.
c.in.srs++
}
//根据消息大小,重新调整所需要的内存空间
// Update read buffer size as/if needed.
if n >= cap(b) && cap(b) < maxBufSize {
// Grow
c.in.rsz = int32(cap(b) * 2)
b = make([]byte, c.in.rsz)
} else if n < cap(b) && cap(b) > minBufSize && c.in.srs > shortsToShrink {
// Shrink, for now don't accelerate, ping/pong will eventually sort it out.
c.in.rsz = int32(cap(b) / 2)
b = make([]byte, c.in.rsz)
}
c.mu.Unlock()
// Check to see if we got closed, e.g. slow consumer
if nc == nil {
return
}
}
}
parse函数较长,主要是从收到的信息里面解析出来几个请求,参考一下剩余代码
func (c *client) parse(buf []byte) error {
// Move to loop instead of range syntax to allow jumping of i
for i = 0; i < len(buf); i++ {
b = buf[i]
switch c.state {
case OP_START:
case PUB_ARG:
switch b {
case '\r':
c.drop = 1
case '\n':
var arg []byte
if c.argBuf != nil {
arg = c.argBuf
c.argBuf = nil
} else {
arg = buf[c.as : i-c.drop]
}
//解析出来发布信息
if err := c.processPub(c.trace, arg); err != nil {
return err
}
c.drop, c.as, c.state = OP_START, i+1, MSG_PAYLOAD
// If we don't have a saved buffer then jump ahead with
// the index. If this overruns what is left we fall out
// and process split buffer.
if c.msgBuf == nil {
i = c.as + c.pa.size - LEN_CR_LF
}
default:
if c.argBuf != nil {
c.argBuf = append(c.argBuf, b)
}
}
case MSG_PAYLOAD:
case MSG_END_R:
case MSG_END_N:
if b != '\n' {
goto parseErr
}
if c.msgBuf != nil {
c.msgBuf = append(c.msgBuf, b)
} else {
c.msgBuf = buf[c.as : i+1]
}
解析出来要扩算的消息
c.processInboundMsg(c.msgBuf)
fmt.Println("*********", string(c.msgBuf))
c.argBuf, c.msgBuf = nil, nil
c.drop, c.as, c.state = 0, i+1, OP_START
// Drop all pub args
c.pa.arg, c.pa.pacache, c.pa.account, c.pa.subject = nil, nil, nil, nil
c.pa.reply, c.pa.szb, c.pa.queues = nil, nil, nil
case OP_A:
case OP_ASUB:
case OP_ASUB_SPC:
case ASUB_ARG:
// 跨集群的订阅
if err := c.processAccountSub(arg); err != nil {
return err
}
c.drop, c.as, c.state = 0, i+1, OP_START
default:
if c.argBuf != nil {
c.argBuf = append(c.argBuf, b)
}
}
case AUSUB_ARG:
switch b {
case '\r':
c.drop = 1
case '\n':
var arg []byte
if c.argBuf != nil {
arg = c.argBuf
c.argBuf = nil
} else {
arg = buf[c.as : i-c.drop]
}
取消跨集群订阅
c.processAccountUnsub(arg)
c.drop, c.as, c.state = 0, i+1, OP_START
}
case SUB_ARG:
switch b {
case '\r':
c.drop = 1
case '\n':
var arg []byte
if c.argBuf != nil {
arg = c.argBuf
c.argBuf = nil
} else {
arg = buf[c.as : i-c.drop]
}
var err error
switch c.kind {
case CLIENT:
本router内的订阅(订阅者和router直接链接通信)
err = c.processSub(arg)
case ROUTER:
别的router收到的订阅信息,扩算到本router中
err = c.processRemoteSub(arg)
case GATEWAY:
别的集群信息扩算到本集群
err = c.processGatewayRSub(arg)
}
if err != nil {
return err
}
c.drop, c.as, c.state = 0, i+1, OP_START
default:
if c.argBuf != nil {
c.argBuf = append(c.argBuf, b)
}
}
case UNSUB_ARG:
switch b {
case '\r':
c.drop = 1
case '\n':
var arg []byte
if c.argBuf != nil {
arg = c.argBuf
c.argBuf = nil
} else {
arg = buf[c.as : i-c.drop]
}
var err error
取消所有订阅
switch c.kind {
case CLIENT:
err = c.processUnsub(arg)
case ROUTER:
err = c.processRemoteUnsub(arg)
case GATEWAY:
err = c.processGatewayRUnsub(arg)
}
if err != nil {
return err
}
c.drop, c.as, c.state = 0, i+1, OP_START
default:
if c.argBuf != nil {
c.argBuf = append(c.argBuf, b)
}
}
case CONNECT_ARG:
switch b {
case '\r':
c.drop = 1
case '\n':
var arg []byte
if c.argBuf != nil {
arg = c.argBuf
c.argBuf = nil
} else {
arg = buf[c.as : i-c.drop]
}
普通消息订阅/构建集群的跨router/构建多集群跨gateway请求(构建链接)
if err := c.processConnect(arg); err != nil {
return err
}
c.drop, c.state = 0, OP_START
// Reset notion on authSet
c.mu.Lock()
authSet = c.awaitingAuth()
c.mu.Unlock()
default:
if c.argBuf != nil {
c.argBuf = append(c.argBuf, b)
}
}
case MSG_ARG:
switch b {
case '\r':
c.drop = 1
case '\n':
var arg []byte
if c.argBuf != nil {
arg = c.argBuf
c.argBuf = nil
} else {
arg = buf[c.as : i-c.drop]
}
接收到 另外一个ROUTER扩散到来的发布消息
if err := c.processRoutedMsgArgs(c.trace, arg); err != nil {
return err
}
c.drop, c.as, c.state = 0, i+1, MSG_PAYLOAD
// jump ahead with the index. If this overruns
// what is left we fall out and process split
// buffer.
i = c.as + c.pa.size - LEN_CR_LF
default:
if c.argBuf != nil {
c.argBuf = append(c.argBuf, b)
}
}
case INFO_ARG:
switch b {
case '\r':
c.drop = 1
case '\n':
var arg []byte
if c.argBuf != nil {
arg = c.argBuf
c.argBuf = nil
} else {
arg = buf[c.as : i-c.drop]
}
链接建立后,协议握手
if err := c.processInfo(arg); err != nil {
return err
}
c.drop, c.as, c.state = 0, i+1, OP_START
default:
if c.argBuf != nil {
c.argBuf = append(c.argBuf, b)
}
}
}
// Check for split buffer scenarios for any ARG state.
if c.state == SUB_ARG || c.state == UNSUB_ARG || c.state == PUB_ARG ||
c.state == ASUB_ARG || c.state == AUSUB_ARG ||
c.state == MSG_ARG || c.state == MINUS_ERR_ARG ||
c.state == CONNECT_ARG || c.state == INFO_ARG {
// Setup a holder buffer to deal with split buffer scenario.
if c.argBuf == nil {
c.argBuf = c.scratch[:0]
c.argBuf = append(c.argBuf, buf[c.as:i-c.drop]...)
}
// Check for violations of control line length here. Note that this is not
// exact at all but the performance hit is too great to be precise, and
// catching here should prevent memory exhaustion attacks.
if len(c.argBuf) > int(mcl) {
c.sendErr("Maximum Control Line Exceeded")
c.closeConnection(MaxControlLineExceeded)
return ErrMaxControlLine
}
}
}
消息来到本地,解析部分到此。作者根据字符一个一个的解析出消息,感觉这个方式可以改变。
消息本地扩散:
根据管理链接的的类型,走各自扩算消息策略
func (c *client) processInboundMsg(msg []byte) { switch c.kind { case CLIENT: c.processInboundClientMsg(msg) case ROUTER: c.processInboundRoutedMsg(msg) case GATEWAY: c.processInboundGatewayMsg(msg) } }
func (c *client) processInboundClientMsg(msg []byte) { if c.opts.Verbose { 收到信息后,给发送端一个回复 c.sendOK() } // Mostly under testing scenarios. if c.srv == nil || c.acc == nil { return } // Match the subscriptions. We will use our own L1 map if // it's still valid, avoiding contention on the shared sublist. var r *SublistResult var ok bool genid := atomic.LoadUint64(&c.acc.sl.genid) if genid == c.in.genid && c.in.results != nil { r, ok = c.in.results[string(c.pa.subject)] } else { // Reset our L1 completely. c.in.results = make(map[string]*SublistResult) c.in.genid = genid } 获取对此消息感兴趣的订阅列表 // Go back to the sublist data structure. if !ok { r = c.acc.sl.Match(string(c.pa.subject)) c.in.results[string(c.pa.subject)] = r // Prune the results cache. Keeps us from unbounded growth. Random delete. if len(c.in.results) > maxResultCacheSize { n := 0 for subject := range c.in.results { delete(c.in.results, subject) if n++; n > pruneSize { break } } } } // Check to see if we need to map/route to another account. if c.acc.imports.services != nil { c.checkForImportServices(c.acc, msg) } var qnames [][]byte // Check for no interest, short circuit if so. // This is the fanout scale. if len(r.psubs)+len(r.qsubs) > 0 { var collect bool // If we have queue subs in this cluster, then if we run in gateway // mode and the remote gateways have queue subs, then we need to // collect the queue groups this message was sent to so that we // exclude them when sending to gateways. if len(r.qsubs) > 0 && c.srv.gateway.enabled && atomic.LoadInt64(&c.srv.gateway.totalQSubs) > 0 { collect = true } 扩算消息函数,后面接着看此函数 qnames = c.processMsgResults(c.acc, r, msg, c.pa.subject, c.pa.reply, collect) } // Now deal with gateways if c.srv.gateway.enabled { // 本地消息,如果配置了GATAWAY,那么消息同步给另外一个集群 c.sendMsgToGateways(c.acc, msg, c.pa.subject, c.pa.reply, qnames) } }
func (c *client) processMsgResults(acc *Account, r *SublistResult, msg, subject, reply []byte, collect bool) [][]byte { var queues [][]byte // msg header for clients. msgh := c.msgb[1:msgHeadProtoLen] msgh = append(msgh, subject...) msgh = append(msgh, ' ') si := len(msgh) // For sending messages across routes. Reset it if we have one. // We reuse this data structure. if c.in.rts != nil { c.in.rts = c.in.rts[:0] } 每个订阅者都需要此消息 for _, sub := range r.psubs { // Check if this is a send to a ROUTER. We now process // these after everything else. if sub.client.kind == ROUTER { if c.kind == ROUTER { //自己是ROUTER,又收到了别的ROUTER同步过来的MSG的时,消息不会再发给其他的router //消息不能无限反复扩散 continue } c.addSubToRouteTargets(sub) continue } else if sub.client.kind == GATEWAY { // Never send to gateway from here. continue } // Check for stream import mapped subs. These apply to local subs only. if sub.im != nil && sub.im.prefix != "" { // Redo the subject here on the fly. msgh = c.msgb[1:msgHeadProtoLen] msgh = append(msgh, sub.im.prefix...) msgh = append(msgh, subject...) msgh = append(msgh, ' ') si = len(msgh) } // Normal delivery mh := c.msgHeader(msgh[:si], sub, reply) c.deliverMsg(sub, mh, msg) } // If we are sourced from a route we need to have direct filtered queues. //本客户端是路由,说明消息是别的ROUTER发来的。处理到此处结束,不再扩散消息 if c.kind == ROUTER && c.pa.queues == nil { return queues } // Set these up to optionally filter based on the queue lists. // This is for messages received from routes which will have directed // guidance on which queue groups we should deliver to. qf := c.pa.queues // For gateway connections, we still want to send messages to routes // even if there are no queue filters. //消息从另外一个集群扩算到本集群,则立即进行本集群内的路由扩散 if c.kind == GATEWAY && qf == nil { goto sendToRoutes } // Check to see if we have our own rand yet. Global rand // has contention with lots of clients, etc. if c.in.prand == nil { c.in.prand = rand.New(rand.NewSource(time.Now().UnixNano())) } // Process queue subs 以下是摘自其他网站的注释。 //排队Queueding 这个术语在不同的上下文有不同的意思,必须仔细区分其用法。 // NATS实现了不支持持久化的分布式队列——通过订阅者的队列组(Queue Group)实现的。 // 订阅者队列组提供了消息交付形式的负载均衡,主题(subject) 订阅既可以是“个体”订阅,又可以是队列组订阅。 // 在创建订阅时,选择加入一个队列组,通过提供一个可选的队列组名。对于个体的 主题(subject) 订阅, // gnatsd服务会尝试交付该 主题(subject) 后续的每一条消息副本给每一个订阅者。 // 而对于队列组的订阅,gnatsd 会尝试交付该 主题(subject) 后续的每一条消息给组中的任意一个订阅者, // 这个选择是随机的。分布式队列的这种交付形式是实时执行的,消息是不会持久化到二级存储中。 // 此外,交付是基于 兴趣图(interest graph)—— 即订阅,所以它不是发布者的操作,而是完全受 gnatsd 的控制。 for i := 0; i < len(r.qsubs); i++ { qsubs := r.qsubs[i] // If we have a filter check that here. We could make this a map or someting more // complex but linear search since we expect queues to be small. Should be faster // and more cache friendly. if qf != nil && len(qsubs) > 0 { tqn := qsubs[0].queue for _, qn := range qf { if bytes.Equal(qn, tqn) { 消息发送给队列中的一个监听者。 goto selectQSub } } continue } selectQSub: // We will hold onto remote qsubs when we are coming from a route // just in case we can no longer do local delivery. var rsub *subscription // Find a subscription that is able to deliver this message // starting at a random index. startIndex := c.in.prand.Intn(len(qsubs)) for i := 0; i < len(qsubs); i++ { index := (startIndex + i) % len(qsubs) sub := qsubs[index] if sub == nil { continue } // Potentially sending to a remote sub across a route. if sub.client.kind == ROUTER { if c.kind == ROUTER { // We just came from a route, so skip and prefer local subs. // Keep our first rsub in case all else fails. if rsub == nil { rsub = sub } continue } else { c.addSubToRouteTargets(sub) if collect { queues = append(queues, sub.queue) } } break } // Check for mapped subs if sub.im != nil && sub.im.prefix != "" { // Redo the subject here on the fly. msgh = c.msgb[1:msgHeadProtoLen] msgh = append(msgh, sub.im.prefix...) msgh = append(msgh, subject...) msgh = append(msgh, ' ') si = len(msgh) } mh := c.msgHeader(msgh[:si], sub, reply) if c.deliverMsg(sub, mh, msg) { // Clear rsub rsub = nil if collect { queues = append(queues, sub.queue) } break } } if rsub != nil { // If we are here we tried to deliver to a local qsub // but failed. So we will send it to a remote. c.addSubToRouteTargets(rsub) if collect { queues = append(queues, rsub.queue) } } } //一个集群内,路由间扩算 sendToRoutes: // If no messages for routes return here. if len(c.in.rts) == 0 { return queues } // We address by index to avoid struct copy. // We have inline structs for memory layout and cache coherency. for i := range c.in.rts { rt := &c.in.rts[i] mh := c.msgb[:msgHeadProtoLen] mh = append(mh, acc.Name...) mh = append(mh, ' ') mh = append(mh, subject...) mh = append(mh, ' ') if len(rt.qs) > 0 { if reply != nil { mh = append(mh, "+ "...) // Signal that there is a reply. mh = append(mh, reply...) mh = append(mh, ' ') } else { mh = append(mh, "| "...) // Only queues } mh = append(mh, rt.qs...) } else if reply != nil { mh = append(mh, reply...) mh = append(mh, ' ') } mh = append(mh, c.pa.szb...) mh = append(mh, _CRLF_...) c.deliverMsg(rt.sub, mh, msg) } return queues }
参考:https://www.linuxidc.com/Linux/2016-10/136418.htm