【kubernetes/k8s源码分析】calico node felix ipset 源码分析

IPSet 描述

       IPSet技术可以将策略中的五元组(协议，源地址，源端口,目的地址，目的端口)合并到有限的集合中，可以大大减少IPTables策略条目从而提高效率。

       -m  set  指定加载的模块：set 对应 IPSet 模块

 

IPSet类型

• hash:ip 单独指定(si=源地址)或(di=目标地址)
• hash:ip,port 指定(sisp=源地址-源端口)、(disp=目的地址-源端口)、(sidp=源地址-目的端口)、(didp=目的地址-目的端口)
• bitmap:port 单独指定端口(sp=源端口)或(dp=目的端口)
• list:set 用于容纳前述三种集合类型，且不能包含本身

 

创建集合

ipset n,create [ SETNAME ] [ TYPENAME ] [ CREATE-OPTIONS ]

SETNAME：即所创建集合的名字

TYPENAME：即类型名字，用类型来储存ip

CREATE-OPTIONS：即创建选项，TYPENAME类型所对应的值

     TYPENAME := method:datatype[,datatype[,datatype]]

     可以使用的method:  bitmap, hash, list

     可以使用的datatype:  ip, net, mac, port, iface

   示例如下创建

[root@master-node tmp]# ipset create zzlin-test hash:ip
[root@master-node tmp]# ipset list

Name: zzlin-test
Type: hash:ip
Revision: 1
Header: family inet hashsize 1024 maxelem 65536
Size in memory: 16528
References: 0
Members:

 

      源代码路径： github.com/projectcalico/felix/ipsets/ipsets.go

IPSets 结构体：

// IPSets manages a whole "plane" of IP sets, i.e. all the IPv4 sets, or all the IPv6 IP sets.
type IPSets struct {
   IPVersionConfig *IPVersionConfig

   ipSetIDToIPSet       map[string]*ipSet
   mainIPSetNameToIPSet map[string]*ipSet

   existingIPSetNames set.Set
   nextTempIPSetIdx   uint

   // dirtyIPSetIDs contains IDs of IP sets that need updating.
   dirtyIPSetIDs  set.Set //
   resyncRequired bool

   // pendingTempIPSetDeletions contains names of temporary IP sets that need to be deleted.  We use it to
   // attempt an early deletion of temporary IP sets, if possible.
   pendingTempIPSetDeletions set.Set
   // pendingIPSetDeletions contains names of IP sets that need to be deleted (including temporary ones).
   pendingIPSetDeletions set.Set

   // Factory for command objects; shimmed for UT mocking.
   newCmd cmdFactory

   // Shim for time.Sleep()
   sleep func(time.Duration)

   gaugeNumIpsets prometheus.Gauge

   logCxt *log.Entry

   // restoreInCopy holds a copy of the stdin that we send to ipset restore.  It is reset
   // after each use.
   restoreInCopy bytes.Buffer
   // stdoutCopy holds a copy of the the stdout emitted by ipset restore. It is reset after
   // each use.
   stdoutCopy bytes.Buffer
   // stderrCopy holds a copy of the the stderr emitted by ipset restore. It is reset after
   // each use.
   stderrCopy bytes.Buffer
}

 

StartDataplaneDriver

        -->  NewIntDataplaneDriver

                   -->  ipsets.NewIPSets

 

1. NewIPSets 实例化

func NewIPSets(ipVersionConfig *IPVersionConfig) *IPSets {
	return NewIPSetsWithShims(
		ipVersionConfig,
		newRealCmd,
		time.Sleep,
	)
}

// NewIPSetsWithShims is an internal test constructor.
func NewIPSetsWithShims(
	ipVersionConfig *IPVersionConfig,
	cmdFactory cmdFactory,
	sleep func(time.Duration),
) *IPSets {

	}
}

 

loopUpdatingDataplane

      -->  apply

             -->  ipSets.ApplyUpdates

 

2.  ApplyUpdates 函数

     重试的时间 retrypDelay 成倍增长，可以看到最多重试 10 次

func (s *IPSets) ApplyUpdates() {
	success := false
	retryDelay := 1 * time.Millisecond
	backOff := func() {
		s.sleep(retryDelay)
		retryDelay *= 2
	}
	for attempt := 0; attempt < 10; attempt++ {
		if attempt > 0 {
			s.logCxt.Info("Retrying after an ipsets update failure...")
		}

    根据 resyncRequired 已经至为 true

// QueueResync forces a resync with the dataplane on the next ApplyUpdates() call.
func (s *IPSets) QueueResync() {
	s.logCxt.Info("Asked to resync with the dataplane on next update.")
	s.resyncRequired = true
}

    2.1 用来修复不一致性的 ipset

if s.resyncRequired {
	// Compare our in-memory state against the dataplane and queue up
	// modifications to fix any inconsistencies.
	s.logCxt.Info("Resyncing ipsets with dataplane.")
	numProblems, err := s.tryResync()
	if err != nil {
		s.logCxt.WithError(err).Warning("Failed to resync with dataplane")
		backOff()
		continue
	}
	if numProblems > 0 {
		s.logCxt.WithField("numProblems", numProblems).Info(
			"Found inconsistencies in dataplane")
	}
	s.resyncRequired = false
}

    2.1.1 tryResync 函数

     可以开到执行的命令 ipset list，如下所示，只展示一个内容

Name: cali40all-ipam-pools
Type: hash:net
Revision: 3
Header: family inet hashsize 1024 maxelem 1048576
Size in memory: 16816
References: 1
Members:
192.170.0.0/16

// tryResync attempts to bring our state into sync with the dataplane.  It scans the contents of the
// IP sets in the dataplane and queues up updates to any IP sets that are out-of-sync.
func (s *IPSets) tryResync() (numProblems int, err error) {
	// Log the time spent as we exit the function.
    ................................
	// As we stream through the data, we extract the name of the IP set and its members. We
	// use the IP set's metadata to convert each member to its canonical form for comparison.
	cmd := s.newCmd("ipset", "list")

    剩下的一大堆就是解析 ipset list 的结果，existingIPSetNames 保存 ipset 表名，以及 members

    2.1.2 tryUpdates 函数

      ipset restore 则用于将导出的内容导入

// tryUpdates attempts to create and/or update IP sets.  It attempts to do the updates as a single
// 'ipset restore' session in order to minimise process forking overhead.  Note: unlike
// 'iptables-restore', 'ipset restore' is not atomic, updates are applied individually.
func (s *IPSets) tryUpdates() error {
	if s.dirtyIPSetIDs.Len() == 0 {
		s.logCxt.Debug("No dirty IP sets.")
		return nil
	}
	// Set up an ipset restore session.
	countNumIPSetCalls.Inc()
	cmd := s.newCmd("ipset", "restore")

      2.1.2.1 迭代 dirtyIPSetIDs 调用 writeUpdates 函数进行更新 ipset 表

      调用命令 ipset create %s %s family %s maxelem %d

// Ask each dirty IP set to write its updates to the stream.
var writeErr error
s.dirtyIPSetIDs.Iter(func(item interface{}) error {
	ipSet := s.ipSetIDToIPSet[item.(string)]
	writeErr = s.writeUpdates(ipSet, stdin)
	if writeErr != nil {
		return set.StopIteration
	}
	return nil
})