K8S-网络插件原理

一前言

CNI和kubelet关系

kubelet负责创建POD，POD启动和删除（可以类比为虚拟机）需要设置网络环境。kubelet自身不关心网络实现，它调用CNI实现。
CNI实现能否同时存在多个？
答案：可以有多个，但是kubelet只会使用第一个有效的插件。如果不同的POD网络插件不一致，它们将无法通信。

二 kubelet启动

image

2.1 kubelet启动命令

kubelet \
    --network-plugin=cni 
    --cni-conf-dir=/etc/cni/net.d 
    --cni-bin-dir=/opt/cni/bin 
    ...

通过--network-plugin指定要使用的网络插件类型。

2.2 Runtime设置

cmd/kubelet/app/server.go:createAndInitKubelet

默认k8s的运行时是Docker。

switch containerRuntime {
case kubetypes.DockerContainerRuntime:
   streamingConfig := getStreamingConfig(kubeCfg, kubeDeps, crOptions)
   ds, err := dockershim.NewDockerService(kubeDeps.DockerClientConfig, crOptions.PodSandboxImage, streamingConfig,
      &pluginSettings, runtimeCgroups, kubeCfg.CgroupDriver, crOptions.DockershimRootDirectory, !crOptions.RedirectContainerStreaming)
  // 省略...
case kubetypes.RemoteContainerRuntime:
   // No-op.
   break
default:
   return nil, fmt.Errorf("unsupported CRI runtime: %q", containerRuntime)
}

2.3 网络插件加载

pkg/kubelet/dockershim/docker_service.go:NewDockerService

网络插件有两种实现cni、kubenet，一般都使用cni。

func NewDockerService(config *ClientConfig, podSandboxImage string, streamingConfig *streaming.Config, pluginSettings *NetworkPluginSettings,
    cgroupsName string, kubeCgroupDriver string, dockershimRootDir string, startLocalStreamingServer bool) (DockerService, error) {

    client := NewDockerClientFromConfig(config)
    // cni插件
    pluginSettings.PluginBinDirs = cni.SplitDirs(pluginSettings.PluginBinDirString)
    cniPlugins := cni.ProbeNetworkPlugins(pluginSettings.PluginConfDir, pluginSettings.PluginCacheDir, pluginSettings.PluginBinDirs)
    // kubenet插件
    cniPlugins = append(cniPlugins, kubenet.NewPlugin(pluginSettings.PluginBinDirs, pluginSettings.PluginCacheDir))
    netHost := &dockerNetworkHost{
        &namespaceGetter{ds},
        &portMappingGetter{ds},
    }
    // 插件初始化
    plug, err := network.InitNetworkPlugin(cniPlugins, pluginSettings.PluginName, netHost, pluginSettings.HairpinMode, pluginSettings.NonMasqueradeCIDR, pluginSettings.MTU)
    // 省略
    return ds, nil
}

三 CNI插件

3.1 插件初始化

func ProbeNetworkPlugins(confDir, cacheDir string, binDirs []string) []network.NetworkPlugin {
    //设置可执行程序路径、配置文件路径等
    plugin := &cniNetworkPlugin{
        defaultNetwork: nil,
        loNetwork:      getLoNetwork(binDirs),
        execer:         utilexec.New(),
        confDir:        confDir,
        binDirs:        binDirs,
        cacheDir:       cacheDir,
    }

    // sync NetworkConfig in best effort during probing.
    plugin.syncNetworkConfig()
    return []network.NetworkPlugin{plugin}
}

3.2 插件实现加载

按照配置文件名称加载实现，找到第一个有效的实现作为CNI实现。

func getDefaultCNINetwork(confDir string, binDirs []string) (*cniNetwork, error) {
    // 从/etc/cni/net.d 查找后缀为conf,conflist,json的配置文件
    files, err := libcni.ConfFiles(confDir, []string{".conf", ".conflist", ".json"})
    cniConfig := &libcni.CNIConfig{Path: binDirs}
    sort.Strings(files)
    for _, confFile := range files {
        var confList *libcni.NetworkConfigList
        if strings.HasSuffix(confFile, ".conflist") {
            confList, err = libcni.ConfListFromFile(confFile)
        } else {
            conf, err := libcni.ConfFromFile(confFile)
            confList, err = libcni.ConfListFromConf(conf)
        }
        // 校验可执行程序是否存在
        caps, err := cniConfig.ValidateNetworkList(context.TODO(), confList)
        return &cniNetwork{
            name:          confList.Name,
            NetworkConfig: confList,
            CNIConfig:     cniConfig,
            Capabilities:  caps,
        }, nil
    }
    return nil, fmt.Errorf("no valid networks found in %s", confDir)
}

3.3 配置

读取配置文件，它的内容如下图所示.

{
      "cniVersion": "0.3.1",
      "name": "my-cni",
      "type": "my-cni",
      "podcidr": "10.244.2.0/24"
}

四网络插件初始化

下面以cni的网络插件分析，cni从配置文件加载用户的网络实现后，开始监控配置是否变更。

func (plugin *cniNetworkPlugin) Init(host network.Host, hairpinMode kubeletconfig.HairpinMode, nonMasqueradeCIDR string, mtu int) error {
    err := plugin.platformInit()
    plugin.host = host
    plugin.syncNetworkConfig()
    // 定时监控配置文件是否发生变更
    go wait.Forever(plugin.syncNetworkConfig, defaultSyncConfigPeriod)
    return nil
}

五插件使用

5.1 容器网络设置

在pod创建后启动之前设置网络信息

func (plugin *cniNetworkPlugin) SetUpPod(namespace string, name string, id kubecontainer.ContainerID, annotations, options map[string]string) error {
    if err := plugin.checkInitialized(); err != nil {
        return err
    }
    netnsPath, err := plugin.host.GetNetNS(id.ID)
    // 添加pod到网络中
    _, err = plugin.addToNetwork(cniTimeoutCtx, plugin.getDefaultNetwork(), name, namespace, id, netnsPath, annotations, options)
    return err
}

func (plugin *cniNetworkPlugin) addToNetwork(ctx context.Context, network *cniNetwork, podName string, podNamespace string, podSandboxID kubecontainer.ContainerID, podNetnsPath string, annotations, options map[string]string) (cnitypes.Result, error) {
    rt, err := plugin.buildCNIRuntimeConf(podName, podNamespace, podSandboxID, podNetnsPath, annotations, options)
    // 调用CNI接口
    res, err := cniNet.AddNetworkList(ctx, netConf, rt)
    return res, nil
}

5.2 CNI

cni关联到用户的实现，从代码中可以看出，自己的CNI实现是一个可执行程序，需要包含ADD等命令。

type CNI interface {
    AddNetworkList(ctx context.Context, net *NetworkConfigList, rt *RuntimeConf) (types.Result, error)
    CheckNetworkList(ctx context.Context, net *NetworkConfigList, rt *RuntimeConf) error
    DelNetworkList(ctx context.Context, net *NetworkConfigList, rt *RuntimeConf) error
    GetNetworkListCachedResult(net *NetworkConfigList, rt *RuntimeConf) (types.Result, error)

    AddNetwork(ctx context.Context, net *NetworkConfig, rt *RuntimeConf) (types.Result, error)
    CheckNetwork(ctx context.Context, net *NetworkConfig, rt *RuntimeConf) error
    DelNetwork(ctx context.Context, net *NetworkConfig, rt *RuntimeConf) error
    GetNetworkCachedResult(net *NetworkConfig, rt *RuntimeConf) (types.Result, error)

    ValidateNetworkList(ctx context.Context, net *NetworkConfigList) ([]string, error)
    ValidateNetwork(ctx context.Context, net *NetworkConfig) ([]string, error)
}

func (c *CNIConfig) addNetwork(ctx context.Context, name, cniVersion string, net *NetworkConfig, prevResult types.Result, rt *RuntimeConf) (types.Result, error) {
    c.ensureExec()
    pluginPath, err := c.exec.FindInPath(net.Network.Type, c.Path)
    newConf, err := buildOneConfig(name, cniVersion, net, prevResult, rt)
    // 调用ADD方法
    return invoke.ExecPluginWithResult(ctx, pluginPath, newConf.Bytes, c.args("ADD", rt), c.exec)
}

// AddNetworkList executes a sequence of plugins with the ADD command
func (c *CNIConfig) AddNetworkList(ctx context.Context, list *NetworkConfigList, rt *RuntimeConf) (types.Result, error) {

    for _, net := range list.Plugins {
        result, err = c.addNetwork(ctx, list.Name, list.CNIVersion, net, result, rt)
    }
    return result, nil
}

六参考

https://www.lijiaocn.com/%E9%A1%B9%E7%9B%AE/2017/05/03/Kubernetes-pod-network.html#%E5%B0%86%E5%AE%B9%E5%99%A8%E5%8A%A0%E5%85%A5%E6%8C%87%E5%AE%9A%E7%BD%91%E7%BB%9C