cni k8s 插件安装_K8S-网络插件原理

一 前言

CNI和kubelet关系

kubelet负责创建POD，POD启动和删除(可以类比为虚拟机)需要设置网络环境。kubelet自身不关心网络实现，它调用CNI实现。

CNI实现能否同时存在多个？

答案：可以有多个，但是kubelet只会使用第一个有效的插件。如果不同的POD网络插件不一致，它们将无法通信。

二 kubelet启动

image

2.1 kubelet启动命令

kubelet \

--network-plugin=cni

--cni-conf-dir=/etc/cni/net.d

--cni-bin-dir=/opt/cni/bin

...

通过--network-plugin指定要使用的网络插件类型。

2.2 Runtime设置

cmd/kubelet/app/server.go:createAndInitKubelet

默认k8s的运行时是Docker。

switch containerRuntime {

case kubetypes.DockerContainerRuntime:

streamingConfig := getStreamingConfig(kubeCfg, kubeDeps, crOptions)

ds, err := dockershim.NewDockerService(kubeDeps.DockerClientConfig, crOptions.PodSandboxImage, streamingConfig,

&pluginSettings, runtimeCgroups, kubeCfg.CgroupDriver, crOptions.DockershimRootDirectory, !crOptions.RedirectContainerStreaming)

// 省略...

case kubetypes.RemoteContainerRuntime:

// No-op.

break

default:

return nil, fmt.Errorf("unsupported CRI runtime: %q", containerRuntime)

}

2.3 网络插件加载

pkg/kubelet/dockershim/docker_service.go:NewDockerService

网络插件有两种实现cni、kubenet，一般都使用cni。

func NewDockerService(config *ClientConfig, podSandboxImage string, streamingConfig *streaming.Config, pluginSettings *NetworkPluginSettings,

cgroupsName string, kubeCgroupDriver string, dockershimRootDir string, startLocalStreamingServer bool) (DockerService, error) {

client := NewDockerClientFromConfig(config)

// cni插件

pluginSettings.PluginBinDirs = cni.SplitDirs(pluginSettings.PluginBinDirString)

cniPlugins := cni.ProbeNetworkPlugins(pluginSettings.PluginConfDir, pluginSettings.PluginCacheDir, pluginSettings.PluginBinDirs)

// kubenet插件

cniPlugins = append(cniPlugins, kubenet.NewPlugin(pluginSettings.PluginBinDirs, pluginSettings.PluginCacheDir))

netHost := &dockerNetworkHost{

&namespaceGetter{ds},

&portMappingGetter{ds},

}

// 插件初始化

plug, err := network.InitNetworkPlugin(cniPlugins, pluginSettings.PluginName, netHost, pluginSettings.HairpinMode, pluginSettings.NonMasqueradeCIDR, pluginSettings.MTU)

// 省略

return ds, nil

}

三 CNI插件

3.1 插件初始化

func ProbeNetworkPlugins(confDir, cacheDir string, binDirs []string) []network.NetworkPlugin {

//设置可执行程序路径、配置文件路径等

plugin := &cniNetworkPlugin{

defaultNetwork: nil,

loNetwork: getLoNetwork(binDirs),

execer: utilexec.New(),

confDir: confDir,

binDirs: binDirs,

cacheDir: cacheDir,

}

// sync NetworkConfig in best effort during probing.

plugin.syncNetworkConfig()

return []network.NetworkPlugin{plugin}

}

3.2 插件实现加载

按照配置文件名称加载实现，找到第一个有效的实现作为CNI实现。

func getDefaultCNINetwork(confDir string, binDirs []string) (*cniNetwork, error) {

// 从/etc/cni/net.d 查找后缀为conf,conflist,json的配置文件

files, err := libcni.ConfFiles(confDir, []string{".conf", ".conflist", ".json"})

cniConfig := &libcni.CNIConfig{Path: binDirs}

sort.Strings(files)

for _, confFile := range files {

var confList *libcni.NetworkConfigList

if strings.HasSuffix(confFile, ".conflist") {

confList, err = libcni.ConfListFromFile(confFile)

} else {

conf, err := libcni.ConfFromFile(confFile)

confList, err = libcni.ConfListFromConf(conf)

}

// 校验可执行程序是否存在

caps, err := cniConfig.ValidateNetworkList(context.TODO(), confList)

return &cniNetwork{

name: confList.Name,

NetworkConfig: confList,

CNIConfig: cniConfig,

Capabilities: caps,

}, nil

}

return nil, fmt.Errorf("no valid networks found in %s", confDir)

}

3.3 配置

读取配置文件，它的内容如下图所示.

{

"cniVersion": "0.3.1",

"name": "my-cni",

"type": "my-cni",

"podcidr": "10.244.2.0/24"

}

四 网络插件初始化

下面以cni的网络插件分析，cni从配置文件加载用户的网络实现后，开始监控配置是否变更。

func (plugin *cniNetworkPlugin) Init(host network.Host, hairpinMode kubeletconfig.HairpinMode, nonMasqueradeCIDR string, mtu int) error {

err := plugin.platformInit()

plugin.host = host

plugin.syncNetworkConfig()

// 定时监控配置文件是否发生变更

go wait.Forever(plugin.syncNetworkConfig, defaultSyncConfigPeriod)

return nil

}

五 插件使用

5.1 容器网络设置

在pod创建后启动之前设置网络信息

func (plugin *cniNetworkPlugin) SetUpPod(namespace string, name string, id kubecontainer.ContainerID, annotations, options map[string]string) error {

if err := plugin.checkInitialized(); err != nil {

return err

}

netnsPath, err := plugin.host.GetNetNS(id.ID)

// 添加pod到网络中

_, err = plugin.addToNetwork(cniTimeoutCtx, plugin.getDefaultNetwork(), name, namespace, id, netnsPath, annotations, options)

return err

}

func (plugin *cniNetworkPlugin) addToNetwork(ctx context.Context, network *cniNetwork, podName string, podNamespace string, podSandboxID kubecontainer.ContainerID, podNetnsPath string, annotations, options map[string]string) (cnitypes.Result, error) {

rt, err := plugin.buildCNIRuntimeConf(podName, podNamespace, podSandboxID, podNetnsPath, annotations, options)

// 调用CNI接口

res, err := cniNet.AddNetworkList(ctx, netConf, rt)

return res, nil

}

5.2 CNI

cni关联到用户的实现，从代码中可以看出，自己的CNI实现是一个可执行程序，需要包含ADD等命令。

type CNI interface {

AddNetworkList(ctx context.Context, net *NetworkConfigList, rt *RuntimeConf) (types.Result, error)

CheckNetworkList(ctx context.Context, net *NetworkConfigList, rt *RuntimeConf) error

DelNetworkList(ctx context.Context, net *NetworkConfigList, rt *RuntimeConf) error

GetNetworkListCachedResult(net *NetworkConfigList, rt *RuntimeConf) (types.Result, error)

AddNetwork(ctx context.Context, net *NetworkConfig, rt *RuntimeConf) (types.Result, error)

CheckNetwork(ctx context.Context, net *NetworkConfig, rt *RuntimeConf) error

DelNetwork(ctx context.Context, net *NetworkConfig, rt *RuntimeConf) error

GetNetworkCachedResult(net *NetworkConfig, rt *RuntimeConf) (types.Result, error)

ValidateNetworkList(ctx context.Context, net *NetworkConfigList) ([]string, error)

ValidateNetwork(ctx context.Context, net *NetworkConfig) ([]string, error)

}

func (c *CNIConfig) addNetwork(ctx context.Context, name, cniVersion string, net *NetworkConfig, prevResult types.Result, rt *RuntimeConf) (types.Result, error) {

c.ensureExec()

pluginPath, err := c.exec.FindInPath(net.Network.Type, c.Path)

newConf, err := buildOneConfig(name, cniVersion, net, prevResult, rt)

// 调用ADD方法

return invoke.ExecPluginWithResult(ctx, pluginPath, newConf.Bytes, c.args("ADD", rt), c.exec)

}

// AddNetworkList executes a sequence of plugins with the ADD command

func (c *CNIConfig) AddNetworkList(ctx context.Context, list *NetworkConfigList, rt *RuntimeConf) (types.Result, error) {

for _, net := range list.Plugins {

result, err = c.addNetwork(ctx, list.Name, list.CNIVersion, net, result, rt)

}

return result, nil

}

六 参考