背景
k8s 1.20 默认采用containerd作为容器运行时
containerd 和 docker兼容问题
- 如果项目中需要用到底层 docker-api 则需要考虑兼容问题
k8s-mon项目 简介
- 是什么：滴滴夜莺Kubernetes monitor
- 项目地址 https://github.com/n9e/k8s-mon
k8s-mon为什么要兼容containerd
因为在k8s-mon中采集容器基础资源指标需要和夜莺的服务树打通
需要获取容器服务树标签N9E_NID默认版本是采用docker inspect接口获取的
package collect
import (
"context"
"github.com/docker/docker/api/types"
"github.com/docker/docker/client"
)
func getLabelMapByDockerSdk(nidLabelName string) (map[string]map[string]string, error) {
ctx := context.Background()
cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
if err != nil {
return nil, err
}
defer cli.Close()
containers, err := cli.ContainerList(ctx, types.ContainerListOptions{})
if err != nil {
return nil, err
}
insM := make(map[string]map[string]string)
whiteKeyM := map[string]struct{}{
nidLabelName: {},
}
for _, container := range containers {
podName := container.Labels["io.kubernetes.pod.name"]
if podName == "" {
continue
}
lastM, loaded := insM[podName]
if !loaded {
lastM = make(map[string]string)
}
for k, v := range container.Labels {
if _, found := whiteKeyM[k]; !found {
continue
}
lastM[k] = v
}
insM[podName] = lastM
}
return insM, nil
}
需要兼容下containerd的接口获取
package collect
import (
"context"
"errors"
"github.com/containerd/containerd"
)
func getLabelMapByContainerdSdk(nidLabelName string) (map[string]map[string]string, error) {
client, err := containerd.New("/run/containerd/containerd.sock", containerd.WithDefaultNamespace("k8s.io"))
if err != nil {
return nil, err
}
defer client.Close()
context := context.Background()
cers, err := client.Containers(context)
if err != nil {
return nil, err
}
if len(cers) == 0 {
return nil, errors.New("got zero containers on this node")
}
insM := make(map[string]map[string]string)
whiteKeyM := map[string]struct{}{
nidLabelName: {},
}
for _, c := range cers {
labels, err := c.Labels(context)
if err != nil {
continue
}
podName := labels["io.kubernetes.pod.name"]
if podName == "" {
continue
}
lastM, loaded := insM[podName]
if !loaded {
lastM = make(map[string]string)
}
for k, v := range labels {
if _, found := whiteKeyM[k]; !found {
continue
}
lastM[k] = v
}
insM[podName] = lastM
}
return insM, nil
}
k8s-mon获取容器tag时需要适配，默认采用docker-api，失败再尝试containerd-api
docker 镜像转成containerd的
ctr 命令执行时需要带上 --namespace k8s.io
docker build
docker build -t k8s-mon:v1 . && docker save k8s-mon > a.tarctr 导入docker 镜像
ctr --namespace k8s.io images import a.tarctr 给镜像打tag = docker tag 4d8690b7763d registry.cn-beijing.aliyuncs.com/n9e/k8s-mon:v2.1.0
ctr --namespace k8s.io images tag docker.io/library/k8s-mon:v1 k8s-mon:v1ctr get镜像 = docker images
[root@k8s-node01 ~]# ctr --namespace k8s.io images ls
REF TYPE DIGEST SIZE PLATFORMS LABELS
docker.io/eipwork/kuboard:latest application/vnd.docker.distribution.manifest.v2+json sha256:7e3d463e856d4d46f88b3241946554b144e3c285953e5055f6a6d6fa2f6360ad 69.0 MiB linux/amd64 io.cri-containerd.image=managed
docker.io/eipwork/kuboard@sha256:7e3d463e856d4d46f88b3241946554b144e3c285953e5055f6a6d6fa2f6360ad application/vnd.docker.distribution.manifest.v2+json sha256:7e3d463e856d4d46f88b3241946554b144e3c285953e5055f6a6d6fa2f6360ad 69.0 MiB linux/amd64 io.cri-containerd.image=managed
docker.io/eipwork/metrics-server:v0.3.7 application/vnd.docker.distribution.manifest.list.v2+json sha256:2a8e43d89e434b6235858444bb0701c44bd03475d2937fdf6831a7bb4ee14240 20.1 MiB linux/amd64,linux/arm64 io.cri-containerd.image=managed
docker.io/eipwork/metrics-server@sha256:2a8e43d89e434b6235858444bb0701c44bd03475d2937fdf6831a7bb4ee14240 application/vnd.docker.distribution.manifest.list.v2+json sha256:2a8e43d89e434b6235858444bb0701c44bd03475d2937fdf6831a7bb4ee14240 20.1 MiB linux/amd64,linux/arm64 io.cri-containerd.image=managed
docker.io/library/k8s-mon:v1 ctr 获取容器 = docker ps
[root@k8s-node01 ~]# ctr --namespace k8s.io c ls
CONTAINER IMAGE RUNTIME
1d79feb8854e66f0c7c21739f7ee048edc1e278ce76d66c52981ea878d6eb231 registry.aliyuncs.com/k8sxio/pause@sha256:927d98197ec1141a368550822d18fa1c60bdae27b78b0c004f705f548c07814f io.containerd.runc.v2
22baad7dc66daf480b8498fe8c31e546bc87f27b3b45807f6c8f942fccc9b686 sha256:919a16510f41bb12c86508529f915b123a5ff589aebb0a4355614a829aa47d62 io.containerd.runc.v2
2d0eefeec5fa5bf6b4bdc41933eb251b9550310b71f1cc300a2d05710cbcd5e7 quay.io/coreos/kube-state-metrics:v1.9.7 io.containerd.runc.v2
492a71bfeba379a6310095c888c21da99f1badb8253a9d57bb4536d4a2e31b82 swr.cn-east-2.myhuaweicloud.com/kuboard-dependency/node@sha256:304dd23bcda5216026f1601cb61395792249f1c58c98771198f6e517b0f5c96b io.containerd.runc.v2
4ebc6262d5217ea9b4f639051cb978398d2bcf38cd68b7bb83a20f270057bc2b sha25
本地build 传输到远端机器
docker build -t k8s-mon:v1 . && docker save k8s-mon > a.tar && nc 172.20.70.215 444 < a.tar
# 远端机器
nc -l 444 > a.tar && docker load < a.tarpush到阿里云
docker push registry.cn-beijing.aliyuncs.com/n9e/k8s-mon:v2.0.7登录到阿里云
docker login --username=燕小乙1875 registry.cn-beijing.aliyuncs.com
删除僵尸容器
docker rmi $(docker images | grep "none" | awk '{print $3}')