【kubernetes】使用client-go与kubernetes集群交互
1 client-go简介及使用流程
client-go是kubernetes官方提供的kubernetes的api库，通过它可以调用kubernetes的api。
使用client-go调用kubernetes分成三个部分：
- 根据kubeconfig创建客户端
- 生成对应的资源对象
- 通过客户端调用接口操作资源对象
调用kubernetes.NewForConfig(c *rest.Config)根据配置对象创建客户端，为了生成rest.Config，有两种方式：一种是调用clientcmd.BuildConfigFromFlags("", *kubeconfig)直接从配置文件读取；另一种是调用clientcmd.RESTConfigFromKubeConfig()从参数的字符串读取。
client-go对资源的操作接口分成两种形式：一种是静态的，操作时需要提供明确的结构体数据；另一种是动态的，操作时只需要提供一个map[string]interface{}的对象。在使用kubectl进行资源的创建、更新、删除操作时，只需要提供yaml文件即可，k8s自己会去解析字段的值，因此，在进行资源的CUD时，客户端只需要提供yaml文件，而资源的查询操作，由于需要获取某个字段的值，而且不同的资源的结构也是不同的，使用动态接口就不太方便。
有了客户端，下面就需要生成对应的资源对象，有两种形式的资源操作接口，如果是用户提交的方式，肯定是用动态的，如果是用程序创建，肯定是用动态的。
动态接口：
dynamicClient, err := dynamic.NewForConfig(restConfig)
if err != nil {
// TODO: handle error
}
// data是从yaml文件读取的数据
// unstructured.Unstructured实际上就是个map[string]interface{}
// 在读取时，考虑到有多个资源对象的情况，因此，使用for循环进行了读取
var resources []unstructured.Unstructured
d := yaml.NewYAMLReader(bufio.NewReader(bytes.NewReader(data)))
for {
data, err := d.Read()
if err != nil {
if err != io.EOF {
return nil, errors.Wrap(err, "decode resource yaml")
}
break
}
var item unstructured.Unstructured
if err := sigsyaml.Unmarshal(data, &item); err != nil {
return nil, errors.Wrap(err, "unmarshal unstructured")
}
kind := item.GetKind()
namespace := item.GetNamespace()
// 根据kind获取对应的Group和Version
group := getGroup(kind)
version := getVersion(kind)
gvr := schema.GroupVersionResource{Group: group, Version: version, Resource: kind}
// 为了减少出现问题的概率，最好的办法是先读取完整个yaml文件，然后再调用接口，
// 否则，如果文件中部分数据不合法，有可能造成部分资源创建成功，部分资源创建失败
_, err := dynamicClient.Resource(gvr).Namespace(namespace).Create(ctx, &item, metav1.CreateOptions{})
if err != nil {
// TODO: handle error
}
}
静态接口：
type Deployment struct {
Kind string
Namespace string
Name string
CreationTimestamp string
Replica int32
AvailableReplicas int32
Images []string
}
k8sClient, err := kubernetes.NewForConfig(restConfig)
if err != nil {
// TODO: handle error
}
itemList, err := k8sClient.AppsV1().Deployments(namespace).List(ctx, metav1.ListOptions{})
if err != nil {
// TODO: handle error
}
// items中保存到就是Deployment的数组，可以返回给用户查看
items := []Deployment{}
for _, v := range itemList.Items {
item := Deployment{
Kind: "Deployment",
Namespace: v.Namespace,
Name: v.Name,
CreationTimestamp: v.CreationTimestamp.Format("2006-01-02 15:04:05"),
Replica: v.Status.Replicas,
AvailableReplicas: v.Status.AvailableReplicas,
}
for _, c := range v.Spec.Template.Spec.Containers {
item.Images = append(item.Images, c.Image)
}
items = append(items, item)
}
2 使用client-go实现控制器
client-go除了调用kubernetes的接口用于资源操作，另一个很重要的场景是实现控制器。
在kubernetes中，控制器是主要逻辑的实现部分，它的工作方式是：
- 监听资源的变化状态，并获取资源的当前状态
- 获取资源的期望状态
- 执行操作让当前状态向期望状态转移
再落实到具体到实现，client-go中也提供了相应的组件：
- Informer：负责跟踪资源的状态变化
- Indexer：在本地存储资源的最新状态
- Workqueue：
因此，使用client-go实现controller的具体逻辑就是：
- Controller通过Informer监听资源状态变化，将资源存储到本地的store中(通过Indexer操作store)
- 如果Informer监听到资源状态发生变化，会调用事先注册的ResourceEventHandler回调函数
- ResourceEventHandler回调函数将关心变更的Object放到Workqueue中
- Controller从Workqueue取出Object，启动worker执行业务逻辑使得当前状态向期望状态转移
- 在worker中使用lister获取resource，不需要频繁访问apiserver
Writing Controllers
workqueue
// Controller demonstrates how to implement a controller with client-go.
type Controller struct {
indexer cache.Indexer //
queue workqueue.RateLimitingInterface // 工作队列
informer cache.Controller // 监听资源变化
}
// 创建控制器
func NewController(queue workqueue.RateLimitingInterface, indexer cache.Indexer, informer cache.Controller) *Controller {
return &Controller{
informer: informer,
indexer: indexer,
queue: queue,
}
}
func (c *Controller) processNextItem() bool {
// 从队列中读取元素
key, quit := c.queue.Get()
if quit {
return false
}
// Tell the queue that we are done with processing this key. This unblocks the key for other workers
// This allows safe parallel processing because two pods with the same key are never processed in
// parallel.
//
defer c.queue.Done(key)
// Invoke the method containing the business logic
err := c.syncToStdout(key.(string))
// Handle the error if something went wrong during the execution of the business logic
c.handleErr(err, key)
return true
}
// 控制器的具体业务逻辑
// 1 根据key使用Indexer从store中获取对应的对象
// 2 如果不存在，则打印相应的信息；如果存在，则打印一条提示信息
// 注意：这里不要执行重试逻辑
func (c *Controller) syncToStdout(key string) error {
obj, exists, err := c.indexer.GetByKey(key)
if err != nil {
klog.Errorf("Fetching object with key %s from store failed with %v", key, err)
return err
}
if !exists {
fmt.Printf("Pod %s does not exist anymore\n", key)
} else {
// 这里需要根据UID判断资源是否是重新创建的
fmt.Printf("Sync/Add/Update for Pod %s\n", obj.(*v1.Pod).GetName())
}
return nil
}
// 如果出错了需要执行重试逻辑
func (c *Controller) handleErr(err error, key interface{}) {
if err == nil {
// Forget about the #AddRateLimited history of the key on every successful synchronization.
// This ensures that future processing of updates for this key is not delayed because of
// an outdated error history.
c.queue.Forget(key)
return
}
// This controller retries 5 times if something goes wrong. After that, it stops trying.
if c.queue.NumRequeues(key) < 5 {
klog.Infof("Error syncing pod %v: %v", key, err)
// Re-enqueue the key rate limited. Based on the rate limiter on the
// queue and the re-enqueue history, the key will be processed later again.
c.queue.AddRateLimited(key)
return
}
c.queue.Forget(key)
// Report to an external entity that, even after several retries, we could not successfully process this key
runtime.HandleError(err)
klog.Infof("Dropping pod %q out of the queue: %v", key, err)
}
// Run begins watching and syncing.
func (c *Controller) Run(workers int, stopCh chan struct{}) {
defer runtime.HandleCrash()
// Let the workers stop when we are done
defer c.queue.ShutDown()
klog.Info("Starting Pod controller")
go c.informer.Run(stopCh)
// Wait for all involved caches to be synced, before processing items from the queue is started
if !cache.WaitForCacheSync(stopCh, c.informer.HasSynced) {
runtime.HandleError(fmt.Errorf("Timed out waiting for caches to sync"))
return
}
for i := 0; i < workers; i++ {
go wait.Until(c.runWorker, time.Second, stopCh)
}
<-stopCh
klog.Info("Stopping Pod controller")
}
func (c *Controller) runWorker() {
for c.processNextItem() {
}
}
func main() {
var kubeconfig string
var master string
flag.StringVar(&kubeconfig, "kubeconfig", "", "absolute path to the kubeconfig file")
flag.StringVar(&master, "master", "", "master url")
flag.Parse()
// 根据配置文件生成配置对象
config, err := clientcmd.BuildConfigFromFlags(master, kubeconfig)
if err != nil {
klog.Fatal(err)
}
// 创建clientset
clientset, err := kubernetes.NewForConfig(config)
if err != nil {
klog.Fatal(err)
}
// 创建POD的监听器
podListWatcher := cache.NewListWatchFromClient(clientset.CoreV1().RESTClient(), "pods", v1.NamespaceDefault, fields.Everything())
// 创建工作队列
queue := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter())
// Bind the workqueue to a cache with the help of an informer. This way we make sure that
// whenever the cache is updated, the pod key is added to the workqueue.
// Note that when we finally process the item from the workqueue, we might see a newer version
// of the Pod than the version which was responsible for triggering the update.
unc(lw cache.ListerWatcher, objType runtime.Object, resyncPeriod time.Duration, h cache.ResourceEventHandler, indexers cache.Indexers) (cache.Indexer, cache.Controller)
NewIndexerInformer returns a Indexer and a controller for populating the index
while also providing event notifications. You should only used the returned
Index for Get/List operations; Add/Modify/Deletes will cause the event
notifications to be faulty.
Parameters:
* lw is list and watch functions for the source of the resource you want to
be informed of.
* objType is an object of the type that you expect to receive.
* resyncPeriod: if non-zero, will re-list this often (you will get OnUpdate
calls, even if nothing changed). Otherwise, re-list will be delayed as
long as possible (until the upstream source closes the watch or times out,
or you stop the controller).
* h is the object you want notifications sent to.
* indexers is the indexer for the received object type.
//
indexer, informer := cache.NewIndexerInformer(podListWatcher, &v1.Pod{}, 0, cache.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) {
key, err := cache.MetaNamespaceKeyFunc(obj)
if err == nil {
queue.Add(key)
}
},
UpdateFunc: func(old interface{}, new interface{}) {
key, err := cache.MetaNamespaceKeyFunc(new)
if err == nil {
queue.Add(key)
}
},
DeleteFunc: func(obj interface{}) {
// IndexerInformer uses a delta queue, therefore for deletes we have to use this
// key function.
key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj)
if err == nil {
queue.Add(key)
}
},
}, cache.Indexers{})
controller := NewController(queue, indexer, informer)
// We can now warm up the cache for initial synchronization.
// Let's suppose that we knew about a pod "mypod" on our last run, therefore add it to the cache.
// If this pod is not there anymore, the controller will be notified about the removal after the
// cache has synchronized.
indexer.Add(&v1.Pod{
ObjectMeta: meta_v1.ObjectMeta{
Name: "mypod",
Namespace: v1.NamespaceDefault,
},
})
// 启动控制器
stop := make(chan struct{})
defer close(stop)
go controller.Run(1, stop)
select {}
}