[k8s源码分析][controller-manager] controller_utils分析

1. 前言

转载请说明原文出处, 尊重他人劳动成果!

源码位置: https://github.com/nicktming/kubernetes/blob/tming-v1.13/pkg/controller/controller_ref_manager.go
分支: tming-v1.13 (基于v1.13版本)

本文将分析controller中的一些公共结构体, 因为很多的controller会共用这些结构体, 所以在分析各种controller之前了解这些结构体的作用很有必要. 本文将主要涉及pkg/controller/controller_utils.go.

2. ControllerExpectationsInterface

architecture.png

该接口会在controller中被用到, 比如在replicaset的controller中会被用来决定是否需要重新计算replicas数与实际数目是否已经相等, 如果不相等, 则需要进行计算需要删除多少pod或者需要增加多少pod等等.

2.1 ControlleeExpectations

type ControllerExpectationsInterface interface {
    GetExpectations(controllerKey string) (*ControlleeExpectations, bool, error)
    SatisfiedExpectations(controllerKey string) bool
    DeleteExpectations(controllerKey string)
    SetExpectations(controllerKey string, add, del int) error
    ExpectCreations(controllerKey string, adds int) error
    ExpectDeletions(controllerKey string, dels int) error
    CreationObserved(controllerKey string)
    DeletionObserved(controllerKey string)
    RaiseExpectations(controllerKey string, add, del int)
    LowerExpectations(controllerKey string, add, del int)
}
type Expectations interface {
    Fulfilled() bool
}
// ControlleeExpectations track controllee creates/deletes.
type ControlleeExpectations struct {
    add       int64
    del       int64
    key       string
    timestamp time.Time
}
func (e *ControlleeExpectations) Add(add, del int64) {
    atomic.AddInt64(&e.add, add)
    atomic.AddInt64(&e.del, del)
}
func (e *ControlleeExpectations) Fulfilled() bool {
    return atomic.LoadInt64(&e.add) <= 0 && atomic.LoadInt64(&e.del) <= 0
}
func (e *ControlleeExpectations) GetExpectations() (int64, int64) {
    return atomic.LoadInt64(&e.add), atomic.LoadInt64(&e.del)
}
func (exp *ControlleeExpectations) isExpired() bool {
    return clock.RealClock{}.Since(exp.timestamp) > ExpectationsTimeout
}

ControlleeExpectations: 用来维护某个key的add和del这两个变量. 另外Fulfilled的条件是这两个都已经小于等于0了.

2.2 ControllerExpectations

type ControllerExpectations struct {
    cache.Store
}
func NewControllerExpectations() *ControllerExpectations {
    return &ControllerExpectations{cache.NewStore(ExpKeyFunc)}
}
func (r *ControllerExpectations) SetExpectations(controllerKey string, add, del int) error {
    exp := &ControlleeExpectations{add: int64(add), del: int64(del), key: controllerKey, timestamp: clock.RealClock{}.Now()}
    klog.V(4).Infof("Setting expectations %#v", exp)
    return r.Add(exp)
}
func (r *ControllerExpectations) DeleteExpectations(controllerKey string) {
    if exp, exists, err := r.GetByKey(controllerKey); err == nil && exists {
        if err := r.Delete(exp); err != nil {
            klog.V(2).Infof("Error deleting expectations for controller %v: %v", controllerKey, err)
        }
    }
}
func (r *ControllerExpectations) GetExpectations(controllerKey string) (*ControlleeExpectations, bool, error) {
    if exp, exists, err := r.GetByKey(controllerKey); err == nil && exists {
        return exp.(*ControlleeExpectations), true, nil
    } else {
        return nil, false, err
    }
}

该结构体ControllerExpectations只有一个变量cache.Store, 用来存储的, 明显存储的是ControlleeExpectations类型.
SetExpectations: 构造了一个ControlleeExpectations方法进行存储起来.
DeleteExpectations: 删除这个key对应的ControlleeExpectations.
GetExpectations: 获取该controllerKey对应的ControlleeExpectations.

接下来看一下它的更新方法, 都是常规操作, 操作store里面的内容.

func (r *ControllerExpectations) ExpectCreations(controllerKey string, adds int) error {
    return r.SetExpectations(controllerKey, adds, 0)
}
func (r *ControllerExpectations) ExpectDeletions(controllerKey string, dels int) error {
    return r.SetExpectations(controllerKey, 0, dels)
}
func (r *ControllerExpectations) LowerExpectations(controllerKey string, add, del int) {
    if exp, exists, err := r.GetExpectations(controllerKey); err == nil && exists {
        exp.Add(int64(-add), int64(-del))
        // The expectations might've been modified since the update on the previous line.
        klog.V(4).Infof("Lowered expectations %#v", exp)
    }
}
func (r *ControllerExpectations) RaiseExpectations(controllerKey string, add, del int) {
    if exp, exists, err := r.GetExpectations(controllerKey); err == nil && exists {
        exp.Add(int64(add), int64(del))
        // The expectations might've been modified since the update on the previous line.
        klog.V(4).Infof("Raised expectations %#v", exp)
    }
}
func (r *ControllerExpectations) CreationObserved(controllerKey string) {
    r.LowerExpectations(controllerKey, 1, 0)
}
func (r *ControllerExpectations) DeletionObserved(controllerKey string) {
    r.LowerExpectations(controllerKey, 0, 1)
}

这里都是更新方法, 方法很简单, 但是每个方法背后的意义还是需要结合真正的controller来进行说明. (后面分析controller的时候会有说明)

这里一个比较重要的方法就是SatisfiedExpectations方法

// 返回true时 代表需要去同步
func (r *ControllerExpectations) SatisfiedExpectations(controllerKey string) bool {
    if exp, exists, err := r.GetExpectations(controllerKey); exists {
        if exp.Fulfilled() {
            klog.V(4).Infof("Controller expectations fulfilled %#v", exp)
            return true
        } else if exp.isExpired() {
            // 已经过期了 需要去同步
            klog.V(4).Infof("Controller expectations expired %#v", exp)
            return true
        } else {
            klog.V(4).Infof("Controller still waiting on expectations %#v", exp)
            return false
        }
    } else if err != nil {
        klog.V(2).Infof("Error encountered while checking expectations %#v, forcing sync", err)
    } else {
        // When a new controller is created, it doesn't have expectations.
        // When it doesn't see expected watch events for > TTL, the expectations expire.
        //  - In this case it wakes up, creates/deletes controllees, and sets expectations again.
        // When it has satisfied expectations and no controllees need to be created/destroyed > TTL, the expectations expire.
        //  - In this case it continues without setting expectations till it needs to create/delete controllees.
        klog.V(4).Infof("Controller %v either never recorded expectations, or the ttl expired.", controllerKey)
    }
    // Trigger a sync if we either encountered and error (which shouldn't happen since we're
    // getting from local store) or this controller hasn't established expectations.
    return true
}

该方法会在分析replicaset的controller时进行说明.

2.3 UIDTrackingControllerExpectations

type UIDTrackingControllerExpectations struct {
    ControllerExpectationsInterface
    uidStoreLock sync.Mutex
    uidStore cache.Store
}
func NewUIDTrackingControllerExpectations(ce ControllerExpectationsInterface) *UIDTrackingControllerExpectations {
    return &UIDTrackingControllerExpectations{ControllerExpectationsInterface: ce, uidStore: cache.NewStore(UIDSetKeyFunc)}
}
type UIDSet struct {
    sets.String
    key string
}

该结构体的设计是为了期望删除的时候需要知道删除哪些元素, 因为ControllerExpectationsInterface中只能用存储的del属性只能是一个数字, 并不能知道具体要删除哪些对象.
所以在此基础上加入UIDSet表明该key具体要删除哪些对象. 因此除了维护ControlleeExpectations之外还要维护UIDSet.

看一下如何存储的, 保存了ControlleeExpectations和UIDSet.

func (u *UIDTrackingControllerExpectations) ExpectDeletions(rcKey string, deletedKeys []string) error {
    u.uidStoreLock.Lock()
    defer u.uidStoreLock.Unlock()
    // 如果以前存在
    if existing := u.GetUIDs(rcKey); existing != nil && existing.Len() != 0 {
        klog.Errorf("Clobbering existing delete keys: %+v", existing)
    }
    expectedUIDs := sets.NewString()
    for _, k := range deletedKeys {
        expectedUIDs.Insert(k)
    }
    klog.V(4).Infof("Controller %v waiting on deletions for: %+v", rcKey, deletedKeys)
    // 维护一个UIDSet
    if err := u.uidStore.Add(&UIDSet{expectedUIDs, rcKey}); err != nil {
        return err
    }
    // 维护ControlleeExpectations
    return u.ControllerExpectationsInterface.ExpectDeletions(rcKey, expectedUIDs.Len())
}
func (u *UIDTrackingControllerExpectations) GetUIDs(controllerKey string) sets.String {
    if uid, exists, err := u.uidStore.GetByKey(controllerKey); err == nil && exists {
        return uid.(*UIDSet).String
    }
    return nil
}

具体删除某一个

func (u *UIDTrackingControllerExpectations) DeletionObserved(rcKey, deleteKey string) {
    u.uidStoreLock.Lock()
    defer u.uidStoreLock.Unlock()

    uids := u.GetUIDs(rcKey)
    if uids != nil && uids.Has(deleteKey) {
        klog.V(4).Infof("Controller %v received delete for pod %v", rcKey, deleteKey)
        //维护ControlleeExpectations
        u.ControllerExpectationsInterface.DeletionObserved(rcKey)
        // 维护UIDSet
        uids.Delete(deleteKey)
    }
}

删除全部某个controllerKey的全部内容.

func (u *UIDTrackingControllerExpectations) DeleteExpectations(rcKey string) {
    u.uidStoreLock.Lock()
    defer u.uidStoreLock.Unlock()

    u.ControllerExpectationsInterface.DeleteExpectations(rcKey)
    if uidExp, exists, err := u.uidStore.GetByKey(rcKey); err == nil && exists {
        if err := u.uidStore.Delete(uidExp); err != nil {
            klog.V(2).Infof("Error deleting uid expectations for controller %v: %v", rcKey, err)
        }
    }
}