k8s informer机制分析

在阅读k8s源码时经常会看到类似的代码片段ctx.InformerFactory.Apps().xxx，比如在创建deploymentcontroller时的代码如下，其会创建三种资源的informer(Deployments, RelicaSets和Pods)，这里的informer是什么东西呢？每调用一次都会创建一个实例吗？

dc, err := deployment.NewDeploymentController(
    ctx.InformerFactory.Apps().V1().Deployments(),
    ctx.InformerFactory.Apps().V1().ReplicaSets(),
    ctx.InformerFactory.Core().V1().Pods(),
    ctx.ClientBuilder.ClientOrDie("deployment-controller"),
)

如下图所示，简单来说informer的作用就是从apiserver获取对象事件，比如增删pod等，保存到本地cache，同时将事件分发到不同的listener，由listener进行特殊处理。

image.png

informer包含多个子模块，reflector用于从apiserver获取对象，并传入deltafifo，controller从deltafifo取出对象，调用sharedinfdexinformer的函数将对象保存到本地cache，同时分发给不同的listener，每个listener是由对对象感兴趣的控制器注册，listerner最后调用控制器的回调函数onUpdate/onAdd/onDelete将对象又插入workqueue。
控制器的主循环中会从workqueue里取出对象，进行自己的处理。

代码结构

下面看一下informer的代码结构, 代码位于client-go中

image.png

本文所说的infomer对应到代码就是结构体sharedIndexInformer，其中的indexer为threadsafe store用来缓存资源对象，process为用来保存对同一个资源感兴趣的多个注册函数，controller用来创建反射器，监听apiserver中资源对象事件。
SharedInformerFactory中最关键的是成员变量informers，其为map类型(比如deployment，replicaset等)，key为资源类型，value为sharedIndexInformer。每个进程只需要创建一个SharedInformerFactory，如果有多个控制器需要监控某个资源，也只需要创建一个此资源对应的sharedIndexInformer，只要注册回调函数即可，当此资源有事件发生时，会调用注册所有的回调函数通知控制器。

源码流程概述

下面为流程概述(为controller-manager进程源码为例)，首先创建SharedInformerFactory，再启动控制器，其中会创建所需的sharedIndexInformer，最后启动SharedInformerFactory，也就是遍历SharedInformerFactory的informers map启动所有的sharedIndexInformer。

a. 创建 SharedInformerFactory
controllerContext, err := CreateControllerContext(c, rootClientBuilder, clientBuilder, ctx.Done())
    sharedInformers := informers.NewSharedInformerFactory(versionedClient, ResyncPeriod(s)())
        NewSharedInformerFactoryWithOptions(client, defaultResync)

b. 遍历 controllerInitializers 启动所有 controller，创建需要的sharedIndexInformer
// StartControllers starts a set of controllers with a specified ControllerContext
StartControllers(controllerContext, startSATokenController, controllerInitializers, unsecuredMux, healthzHandler);
    //遍历所有的controller
    for controllerName, initFn := range controllers {
        klog.V(1).Infof("Starting %q", controllerName)
        //调用 controller 的初始化函数，比如 startDeploymentController
        ctrl, started, err := initFn(ctx)
            dc, err := deployment.NewDeploymentController(
                ctx.InformerFactory.Apps().V1().Deployments(),
                ctx.InformerFactory.Apps().V1().ReplicaSets(),
                ctx.InformerFactory.Core().V1().Pods(),
                ctx.ClientBuilder.ClientOrDie("deployment-controller"),
            )
                dInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
                    AddFunc:    dc.addDeployment,
                    UpdateFunc: dc.updateDeployment,
                    // This will enter the sync loop and no-op, because the deployment has been deleted from the store.
                    DeleteFunc: dc.deleteDeployment,
                })
                    s.AddEventHandlerWithResyncPeriod(handler, s.defaultEventHandlerResyncPeriod)
                            listener := newProcessListener(handler, resyncPeriod, determineResyncPeriod(resyncPeriod, s.resyncCheckPeriod), s.clock.Now(), initialBufferSize)
                            s.processor.addListener(listener)
                                p.addListenerLocked(listener)
                                if p.listenersStarted {
                                    //启动协程
                                    p.wg.Start(listener.run)
                                    //启动协程
                                    p.wg.Start(listener.pop)
                                }
            go dc.Run(int(ctx.ComponentConfig.DeploymentController.ConcurrentDeploymentSyncs), ctx.Stop)
                for i := 0; i < workers; i++ {
                    go wait.Until(dc.worker, time.Second, stopCh)
                        for dc.processNextWorkItem() {
                        }
                            key, quit := dc.queue.Get()
                            if quit {
                                return false
                            }
                            defer dc.queue.Done(key)

                            //dc.syncHandler 为 dc.syncDeployment
                            err := dc.syncHandler(key.(string))
                }
    }

c. 启动 SharedInformerFactory
controllerContext.InformerFactory.Start(controllerContext.Stop)
    for informerType, informer := range f.informers {
        if !f.startedInformers[informerType] {
            //调用 sharedIndexInformer 的 Run 函数
            go informer.Run(stopCh)
                s.controller = New(cfg)
                wg.StartWithChannel(processorStopCh, s.processor.run)
                    for _, listener := range p.listeners {
                        p.wg.Start(listener.run)
                        p.wg.Start(listener.pop)
                    }
                s.controller.Run(stopCh)
                    r := NewReflector(
                        c.config.ListerWatcher,
                        c.config.ObjectType,
                        c.config.Queue,
                        c.config.FullResyncPeriod,
                    )
                    //启动 reflector 协程
                    wg.StartWithChannel(stopCh, r.Run)
                        if err := r.ListAndWatch(stopCh); err != nil {
                            r.watchErrorHandler(r, err)
                                r.watchHandler(start, w, &resourceVersion, resyncerrc, stopCh);
                                    switch event.Type {
                                    case watch.Added:
                                        //store 为 c.config.Queue，将事件添加到 deltafifo
                                        err := r.store.Add(event.Object)
                                        ...
                                    }
                        }
                    wait.Until(c.processLoop, time.Second, stopCh)
                        for {
                            //c.config.Process 为 s.HandleDeltas
                            //从 deltafifo c.config.Queue 取出事件
                            obj, err := c.config.Queue.Pop(PopProcessFunc(c.config.Process))
                                s.indexer.Add(d.Object);
                                //将事件分发给多个 listener
                                s.processor.distribute(addNotification{newObj: d.Object}, false)
                                    listener.add(obj)
                                        p.addCh <- notification

                                        func (p *processorListener) pop()
                                            //将 notification 又加入通道 nextCh
                                            case nextCh <- notification:

                                        func (p *processorListener) run()
                                            //OnAdd为控制器注册的回调函数，比如deployment控制器的addDeployment
                                            p.handler.OnAdd(notification.newObj)
                                                dc.enqueueDeployment(d)
                                                    dc.queue.Add(key)
                        }
            f.startedInformers[informerType] = true
        }
    }

源码详细流程

a. 创建 NewSharedInformerFactory

type sharedInformerFactory struct {
    client           kubernetes.Interface
    namespace        string
    tweakListOptions internalinterfaces.TweakListOptionsFunc
    lock             sync.Mutex
    defaultResync    time.Duration
    customResync     map[reflect.Type]time.Duration

    informers map[reflect.Type]cache.SharedIndexInformer
    // startedInformers is used for tracking which informers have been started.
    // This allows Start() to be called multiple times safely.
    startedInformers map[reflect.Type]bool
}

//cmd/controller-manager/app/controllermanager.go
controllerContext, err := CreateControllerContext(c, rootClientBuilder, clientBuilder, ctx.Done())
    sharedInformers := informers.NewSharedInformerFactory(versionedClient, ResyncPeriod(s)())
    ctx := ControllerContext{
        ClientBuilder:                   clientBuilder,
        InformerFactory:                 sharedInformers,
        ...
    }

//client-go/informers/factory.go
// NewSharedInformerFactory constructs a new instance of sharedInformerFactory for all namespaces.
func NewSharedInformerFactory(client kubernetes.Interface, defaultResync time.Duration) SharedInformerFactory {
    return NewSharedInformerFactoryWithOptions(client, defaultResync)
}

// NewSharedInformerFactoryWithOptions constructs a new instance of a SharedInformerFactory with additional options.
func NewSharedInformerFactoryWithOptions(client kubernetes.Interface, defaultResync time.Duration, options ...SharedInformerOption) SharedInformerFactory {
    factory := &sharedInformerFactory{
        client:           client,
        namespace:        v1.NamespaceAll,
        defaultResync:    defaultResync,
        informers:        make(map[reflect.Type]cache.SharedIndexInformer),
        startedInformers: make(map[reflect.Type]bool),
        customResync:     make(map[reflect.Type]time.Duration),
    }

    // Apply all options
    for _, opt := range options {
        factory = opt(factory)
    }

    return factory
}

b. 获取informer

启动各种controller，这里以deploymentcontroller为例
func startDeploymentController(ctx ControllerContext) (controller.Interface, bool, error)
    dc, err := deployment.NewDeploymentController(
        ctx.InformerFactory.Apps().V1().Deployments(),
        ctx.InformerFactory.Apps().V1().ReplicaSets(),
        ctx.InformerFactory.Core().V1().Pods(),
        ctx.ClientBuilder.ClientOrDie("deployment-controller"),
    )

    go dc.Run(int(ctx.ComponentConfig.DeploymentController.ConcurrentDeploymentSyncs), ctx.Stop)

    return nil, true, nil

2.1 ctx.InformerFactory.Apps().V1().Deployments() 的实现
//client-go/informers/factory.go
//ctx.InformerFactory.Apps()
func (f *sharedInformerFactory) Apps() apps.Interface {
    return apps.New(f, f.namespace, f.tweakListOptions)
}

//client-go/informers/apps/interface.go
// New returns a new Interface.
func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface {
    return &group{factory: f, namespace: namespace, tweakListOptions: tweakListOptions}
}

//ctx.InformerFactory.Apps().V1()
//client-go/informers/apps/interface.go
// V1 returns a new v1.Interface.
func (g *group) V1() v1.Interface {
    return v1.New(g.factory, g.namespace, g.tweakListOptions)
}

//client-go/informers/apps/v1/interface.go
// New returns a new Interface.
func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface {
    return &version{factory: f, namespace: namespace, tweakListOptions: tweakListOptions}
}

//client-go/informers/apps/v1/interface.go
// Deployments returns a DeploymentInformer.
func (v *version) Deployments() DeploymentInformer {
    return &deploymentInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions}
}

//client-go/informers/apps/v1/deployment.go
type deploymentInformer struct {
    factory          internalinterfaces.SharedInformerFactory
    tweakListOptions internalinterfaces.TweakListOptionsFunc
    namespace        string
}

2.2 初始化 DeploymentController
func NewDeploymentController(dInformer appsinformers.DeploymentInformer, rsInformer appsinformers.ReplicaSetInformer, podInformer coreinformers.PodInformer, client clientset.Interface) (*DeploymentController, error) {
    dc := &DeploymentController{
        client:        client,
        eventRecorder: eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "deployment-controller"}),
        queue:         workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "deployment"),
    }
    
    dInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
        AddFunc:    dc.addDeployment,
        UpdateFunc: dc.updateDeployment,
        // This will enter the sync loop and no-op, because the deployment has been deleted from the store.
        DeleteFunc: dc.deleteDeployment,
    })
    
    rsInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
        AddFunc:    dc.addReplicaSet,
        UpdateFunc: dc.updateReplicaSet,
        DeleteFunc: dc.deleteReplicaSet,
    })

    podInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
        DeleteFunc: dc.deletePod,
    })
    
2.2.1 dInformer.Informer()创建并返回 SharedIndexInformer
//client-go/informers/apps/v1/deployment.go
func (f *deploymentInformer) Informer() cache.SharedIndexInformer {
    return f.factory.InformerFor(&extensionsv1beta1.Deployment{}, f.defaultInformer)
        //client-go/informers/factory.go
        f.lock.Lock()
        defer f.lock.Unlock()

        informerType := reflect.TypeOf(obj)
        //如果已经存在 deployment 类型的informer，则将其返回即可，不用再创建。
        //这也就是为什么称为shared的原因
        informer, exists := f.informers[informerType]
        if exists {
            return informer
        }

        resyncPeriod, exists := f.customResync[informerType]
        if !exists {
            resyncPeriod = f.defaultResync
        }

        //newFunc 为 f.defaultInformer
        informer = newFunc(f.client, resyncPeriod)
        f.informers[informerType] = informer
        
        return informer
}

func (f *deploymentInformer) Lister() v1beta1.DeploymentLister {
    return v1beta1.NewDeploymentLister(f.Informer().GetIndexer())
}

func (f *deploymentInformer) defaultInformer(client kubernetes.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer {
    return NewFilteredDeploymentInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions)
}

// NewFilteredDeploymentInformer constructs a new informer for Deployment type.
// Always prefer using an informer factory to get a shared informer instead of getting an independent
// one. This reduces memory footprint and number of connections to the server.
func NewFilteredDeploymentInformer(client kubernetes.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer {
    return cache.NewSharedIndexInformer(
        &cache.ListWatch{
            ListFunc: func(options v1.ListOptions) (runtime.Object, error) {
                if tweakListOptions != nil {
                    tweakListOptions(&options)
                }
                return client.ExtensionsV1beta1().Deployments(namespace).List(context.TODO(), options)
            },
            WatchFunc: func(options v1.ListOptions) (watch.Interface, error) {
                if tweakListOptions != nil {
                    tweakListOptions(&options)
                }
                return client.ExtensionsV1beta1().Deployments(namespace).Watch(context.TODO(), options)
            },
        },
        &extensionsv1beta1.Deployment{},
        resyncPeriod,
        indexers,
    )
}

func NewSharedIndexInformer(lw ListerWatcher, exampleObject runtime.Object, defaultEventHandlerResyncPeriod time.Duration, indexers Indexers) SharedIndexInformer {
    realClock := &clock.RealClock{}
    sharedIndexInformer := &sharedIndexInformer{
        processor:                       &sharedProcessor{clock: realClock},
        indexer:                         NewIndexer(DeletionHandlingMetaNamespaceKeyFunc, indexers),
        listerWatcher:                   lw,
        objectType:                      exampleObject,
        resyncCheckPeriod:               defaultEventHandlerResyncPeriod,
        defaultEventHandlerResyncPeriod: defaultEventHandlerResyncPeriod,
        cacheMutationDetector:           NewCacheMutationDetector(fmt.Sprintf("%T", exampleObject)),
        clock:                           realClock,
    }
    return sharedIndexInformer
}

2.2.2 AddEventHandler 添加事件处理函数
func (s *sharedIndexInformer) AddEventHandler(handler ResourceEventHandler) {
    s.AddEventHandlerWithResyncPeriod(handler, s.defaultEventHandlerResyncPeriod)
}

func (s *sharedIndexInformer) AddEventHandlerWithResyncPeriod(handler ResourceEventHandler, resyncPeriod time.Duration) {
    s.startedLock.Lock()
    defer s.startedLock.Unlock()

    ...

    listener := newProcessListener(handler, resyncPeriod, determineResyncPeriod(resyncPeriod, s.resyncCheckPeriod), s.clock.Now(), initialBufferSize)
        ret := &processorListener{
            nextCh:                make(chan interface{}),
            addCh:                 make(chan interface{}),
            handler:               handler,
            pendingNotifications:  *buffer.NewRingGrowing(bufferSize),
            requestedResyncPeriod: requestedResyncPeriod,
            resyncPeriod:          resyncPeriod,
        }
        ret.determineNextResync(now)

    if !s.started {
        s.processor.addListener(listener)
        return
    }

    // in order to safely join, we have to
    // 1. stop sending add/update/delete notifications
    // 2. do a list against the store
    // 3. send synthetic "Add" events to the new handler
    // 4. unblock
    s.blockDeltas.Lock()
    defer s.blockDeltas.Unlock()

    s.processor.addListener(listener)
    for _, item := range s.indexer.List() {
        listener.add(addNotification{newObj: item})
    }
}

func (p *sharedProcessor) addListener(listener *processorListener) {
    p.listenersLock.Lock()
    defer p.listenersLock.Unlock()

    p.addListenerLocked(listener)
    if p.listenersStarted {
        //启动协程
        p.wg.Start(listener.run)
        //启动协程
        p.wg.Start(listener.pop)
    }
}

func (p *sharedProcessor) addListenerLocked(listener *processorListener) {
    p.listeners = append(p.listeners, listener)
    p.syncingListeners = append(p.syncingListeners, listener)
}

c. start SharedInformerFactory

controllerContext.InformerFactory.Start(controllerContext.Stop)

// Start initializes all requested informers.
func (f *sharedInformerFactory) Start(stopCh <-chan struct{}) {
    f.lock.Lock()
    defer f.lock.Unlock()

    //遍历所有的 SharedInformerFactory，执行其run函数
    for informerType, informer := range f.informers {
        if !f.startedInformers[informerType] {
            //调用 sharedIndexInformer 的 Run 函数
            go informer.Run(stopCh)
            f.startedInformers[informerType] = true
        }
    }
}

func (s *sharedIndexInformer) Run(stopCh <-chan struct{}) {
    defer utilruntime.HandleCrash()

    fifo := NewDeltaFIFOWithOptions(DeltaFIFOOptions{
        KnownObjects:          s.indexer,
        EmitDeltaTypeReplaced: true,
    })

    cfg := &Config{
        Queue:            fifo,
        ListerWatcher:    s.listerWatcher,
        ObjectType:       s.objectType,
        FullResyncPeriod: s.resyncCheckPeriod,
        RetryOnError:     false,
        ShouldResync:     s.processor.shouldResync,

        Process:           s.HandleDeltas,
        WatchErrorHandler: s.watchErrorHandler,
    }

    func() {
        s.startedLock.Lock()
        defer s.startedLock.Unlock()

        s.controller = New(cfg)
        s.controller.(*controller).clock = s.clock
        s.started = true
    }()

    // Separate stop channel because Processor should be stopped strictly after controller
    processorStopCh := make(chan struct{})
    var wg wait.Group
    defer wg.Wait()              // Wait for Processor to stop
    defer close(processorStopCh) // Tell Processor to stop
    wg.StartWithChannel(processorStopCh, s.cacheMutationDetector.Run)
    //启动 sharedprocessor
    wg.StartWithChannel(processorStopCh, s.processor.run)

    defer func() {
        s.startedLock.Lock()
        defer s.startedLock.Unlock()
        s.stopped = true // Don't want any new listeners
    }()
    s.controller.Run(stopCh)
}

//启动 sharedprocessor，执行listener的run和pop，其中pop用来从addCh通道获取对象，
//并传入nextCH通道，run用来从nextCh通道获取对象，执行控制器注册的回调函数
func (p *sharedProcessor) run(stopCh <-chan struct{}) {
    func() {
        p.listenersLock.RLock()
        defer p.listenersLock.RUnlock()
        for _, listener := range p.listeners {
            p.wg.Start(listener.run)
            p.wg.Start(listener.pop)
        }
        p.listenersStarted = true
    }()
    <-stopCh
    p.listenersLock.RLock()
    defer p.listenersLock.RUnlock()
    for _, listener := range p.listeners {
        close(listener.addCh) // Tell .pop() to stop. .pop() will tell .run() to stop
    }
    p.wg.Wait() // Wait for all .pop() and .run() to stop
}

//创建controller
// New makes a new Controller from the given Config.
func New(c *Config) Controller {
    ctlr := &controller{
        config: *c,
        clock:  &clock.RealClock{},
    }
    return ctlr
}

//执行controller，会创建reflector协程，用来监听资源事件
func (c *controller) Run(stopCh <-chan struct{}) {
    defer utilruntime.HandleCrash()
    go func() {
        <-stopCh
        c.config.Queue.Close()
    }()
    //创建反射器
    r := NewReflector(
        c.config.ListerWatcher,
        c.config.ObjectType,
        c.config.Queue,
        c.config.FullResyncPeriod,
    )
    r.ShouldResync = c.config.ShouldResync
    r.WatchListPageSize = c.config.WatchListPageSize
    r.clock = c.clock
    if c.config.WatchErrorHandler != nil {
        r.watchErrorHandler = c.config.WatchErrorHandler
    }

    c.reflectorMutex.Lock()
    c.reflector = r
    c.reflectorMutex.Unlock()

    var wg wait.Group

    //启动 reflector 协程
    wg.StartWithChannel(stopCh, r.Run)

    wait.Until(c.processLoop, time.Second, stopCh)
    wg.Wait()
}

//reflector 协程，用来监听资源事件，如果有事件发生，则添加到deltafifo
func (r *Reflector) Run(stopCh <-chan struct{}) {
    klog.V(3).Infof("Starting reflector %s (%s) from %s", r.expectedTypeName, r.resyncPeriod, r.name)
    wait.BackoffUntil(func() {
        if err := r.ListAndWatch(stopCh); err != nil {
            r.watchErrorHandler(r, err)
                r.watchHandler(start, w, &resourceVersion, resyncerrc, stopCh);
                    switch event.Type {
                    case watch.Added:
                        //store 为 c.config.Queue，将事件添加到 deltafifo
                        err := r.store.Add(event.Object)
                        ...
                    }
        }
    }, r.backoffManager, true, stopCh)
    klog.V(3).Infof("Stopping reflector %s (%s) from %s", r.expectedTypeName, r.resyncPeriod, r.name)
}

//controller主循环，从 deltafifo 获取资源事件
func (c *controller) processLoop() {
    for {
        //c.config.Process 为 s.HandleDeltas
        //从 deltafifo c.config.Queue 取出事件
        obj, err := c.config.Queue.Pop(PopProcessFunc(c.config.Process))
        ...
    }
}

func (f *DeltaFIFO) Pop(process PopProcessFunc) (interface{}, error) {
    f.lock.Lock()
    defer f.lock.Unlock()
    for {
        for len(f.queue) == 0 {
            // When the queue is empty, invocation of Pop() is blocked until new item is enqueued.
            // When Close() is called, the f.closed is set and the condition is broadcasted.
            // Which causes this loop to continue and return from the Pop().
            if f.closed {
                return nil, ErrFIFOClosed
            }

            f.cond.Wait()
        }
        id := f.queue[0]
        f.queue = f.queue[1:]
        depth := len(f.queue)
        if f.initialPopulationCount > 0 {
            f.initialPopulationCount--
        }
        item, ok := f.items[id]
        ...
        Process 为 s.HandleDeltas
        err := process(item)

        // Don't need to copyDeltas here, because we're transferring
        // ownership to the caller.
        return item, err
    }
}

func (s *sharedIndexInformer) HandleDeltas(obj interface{}) error {
    s.blockDeltas.Lock()
    defer s.blockDeltas.Unlock()

    // from oldest to newest
    for _, d := range obj.(Deltas) {
        switch d.Type {
        case Sync, Replaced, Added, Updated:
            s.cacheMutationDetector.AddObject(d.Object)
            if old, exists, err := s.indexer.Get(d.Object); err == nil && exists {
                ...
            } else {
                if err := s.indexer.Add(d.Object); err != nil {
                    return err
                }
                //将事件分发给多个 listener
                s.processor.distribute(addNotification{newObj: d.Object}, false)
            }
        case Deleted:
            if err := s.indexer.Delete(d.Object); err != nil {
                return err
            }
            s.processor.distribute(deleteNotification{oldObj: d.Object}, false)
        }
    }

    return nil
}

//将事件分发给多个 listener
func (p *sharedProcessor) distribute(obj interface{}, sync bool) {
    p.listenersLock.RLock()
    defer p.listenersLock.RUnlock()

    if sync {
        for _, listener := range p.syncingListeners {
            listener.add(obj)
        }
    } else {
        for _, listener := range p.listeners {
            listener.add(obj)
        }
    }
}

//将事件对象加入通道 addCh
func (p *processorListener) add(notification interface{}) {
    p.addCh <- notification
}

//协程，从通道 addCh 取出事件对象 notification
func (p *processorListener) pop() {
    defer utilruntime.HandleCrash()
    defer close(p.nextCh) // Tell .run() to stop

    var nextCh chan<- interface{}
    var notification interface{}
    for {
        select {
        //将 notification 又加入通道 nextCh
        case nextCh <- notification:
            // Notification dispatched
            var ok bool
            notification, ok = p.pendingNotifications.ReadOne()
            if !ok { // Nothing to pop
                nextCh = nil // Disable this select case
            }
        case notificationToAdd, ok := <-p.addCh:
            if !ok {
                return
            }
            if notification == nil { // No notification to pop (and pendingNotifications is empty)
                // Optimize the case - skip adding to pendingNotifications
                notification = notificationToAdd
                nextCh = p.nextCh
            } else { // There is already a notification waiting to be dispatched
                p.pendingNotifications.WriteOne(notificationToAdd)
            }
        }
    }
}

//协程，从通道 nextCh 取出 notification
func (p *processorListener) run() {
    // this call blocks until the channel is closed.  When a panic happens during the notification
    // we will catch it, **the offending item will be skipped!**, and after a short delay (one second)
    // the next notification will be attempted.  This is usually better than the alternative of never
    // delivering again.
    stopCh := make(chan struct{})
    wait.Until(func() {
        for next := range p.nextCh {
            //根据事件类型，调用不同的处理函数，比如 addDeployment
            switch notification := next.(type) {
            case updateNotification:
                p.handler.OnUpdate(notification.oldObj, notification.newObj)
            case addNotification:
                //比如 addDeployment
                p.handler.OnAdd(notification.newObj)
            case deleteNotification:
                p.handler.OnDelete(notification.oldObj)
            default:
                utilruntime.HandleError(fmt.Errorf("unrecognized notification: %T", next))
            }
        }
        // the only way to get here is if the p.nextCh is empty and closed
        close(stopCh)
    }, 1*time.Second, stopCh)
}

//DeploymentController 的处理函数 addDeployment
func (dc *DeploymentController) addDeployment(obj interface{}) {
    d := obj.(*apps.Deployment)
    klog.V(4).InfoS("Adding deployment", "deployment", klog.KObj(d))
    //dc.enqueue
    dc.enqueueDeployment(d)
}

func (dc *DeploymentController) enqueue(deployment *apps.Deployment) {
    key, err := controller.KeyFunc(deployment)

    //将key插入workqueue dc.queue
    dc.queue.Add(key)
}

//协程用于从 dc.queue 接收消息进行处理
// Run begins watching and syncing.
func (dc *DeploymentController) Run(workers int, stopCh <-chan struct{}) {
    defer utilruntime.HandleCrash()
    defer dc.queue.ShutDown()

    klog.InfoS("Starting controller", "controller", "deployment")
    defer klog.InfoS("Shutting down controller", "controller", "deployment")

    if !cache.WaitForNamedCacheSync("deployment", stopCh, dc.dListerSynced, dc.rsListerSynced, dc.podListerSynced) {
        return
    }

    for i := 0; i < workers; i++ {
        go wait.Until(dc.worker, time.Second, stopCh)
    }

    <-stopCh
}

// worker runs a worker thread that just dequeues items, processes them, and marks them done.
// It enforces that the syncHandler is never invoked concurrently with the same key.
func (dc *DeploymentController) worker() {
    for dc.processNextWorkItem() {
    }
}

func (dc *DeploymentController) processNextWorkItem() bool {
    key, quit := dc.queue.Get()
    if quit {
        return false
    }
    defer dc.queue.Done(key)

    //dc.syncHandler 为 dc.syncDeployment
    err := dc.syncHandler(key.(string))
    dc.handleErr(err, key)

    return true
}

func (dc *DeploymentController) syncDeployment(key string) error {
    namespace, name, err := cache.SplitMetaNamespaceKey(key)
    
    //dc.dLister = dInformer.Lister()
    deployment, err := dc.dLister.Deployments(namespace).Get(name)
    ...
}

func (f *deploymentInformer) Lister() v1beta1.DeploymentLister {
    //返回 cache.SharedIndexInformer 的 indexer
    return v1beta1.NewDeploymentLister(f.Informer().GetIndexer())
}

// NewDeploymentLister returns a new DeploymentLister.
func NewDeploymentLister(indexer cache.Indexer) DeploymentLister {
    return &deploymentLister{indexer: indexer}
}

// deploymentLister implements the DeploymentLister interface.
type deploymentLister struct {
    indexer cache.Indexer
}

// Deployments returns an object that can list and get Deployments.
func (s *deploymentLister) Deployments(namespace string) DeploymentNamespaceLister {
    return deploymentNamespaceLister{indexer: s.indexer, namespace: namespace}
}

// Get retrieves the Deployment from the indexer for a given namespace and name.
func (s deploymentNamespaceLister) Get(name string) (*v1beta1.Deployment, error) {
    obj, exists, err := s.indexer.GetByKey(s.namespace + "/" + name)
    if err != nil {
        return nil, err
    }
    if !exists {
        return nil, errors.NewNotFound(v1beta1.Resource("deployment"), name)
    }
    return obj.(*v1beta1.Deployment), nil
}