基于kubernetes Api拉取deployment service

由于现有工作的需要，需要对python以及kubernetes的使用更为熟悉，所有尝试利用Python完成kubernetes的一系列操作。

此脚本是利用调用kubernetes的Api接口完成对deployment,与service的拉取，从而实现对该yaml的一个备份处理。

由于日常开发反馈一些问题，我们运维会对一些deploy，service进行命令行的修改，使得配置文件中与实际kubernetes运用中的有一定出入，所以6.0更新了将得到的kubernetes中最新的内容，通过重新赋值的方式更新该部分数据，从而实现最真实的备份。

                                                  最近更新时间：2019.11.15

# ！/usr/bin/env python3
# -*- coding: utf-8 -*-
# @Author  : jiangjw
''' 可实现拉取kubernetes中的deployment与service,并组合为一个完整的yml文件；
    4.0 更新了创建当天对应时间目录，并将各个team，以及cloud以目录分隔开来；
    5.0 更新了删除7天以前的备份目录。
    6.0 将内存中的镜像、ports等数据重新赋值，达到该文件为当前kubernetes实际应用的状态。'''

import os, yaml, json, datetime, shutil
from time import sleep
from tqdm import tqdm  # 进度条模块
from kubernetes import client, config
from kubernetes.client.rest import ApiException


def deployment_all(namespace, timedir, data):
    api_instance_deploy = client.AppsV1Api()
    k8s_host = [
        {"ip": "192.168.1.20",
         "hostnames": [
             "xxx.xxx.net", "xxx.xxx.net", "xxx.xxx.net",
             "xxx.xxx.net", "xxx.xxx.net",
             "xxx.xxx.net", "xxx.xxx.net", "xxx.xxx.net",
             "xxx.xxx.net", "xxx.xxx.net", "xxx.xxx.net",
             "xxx.xxx.net"]
         }
    ]
    k8s_dns = {
        "options": [
            {"name": "ndots", "value": "2"},
            {"name": "single-request-reopen"}
        ]
    }
    deploy_success_num = 0
    deploy_fail_num = 0
    deploy_fail_name = []
    team_list1 = []

    try:
        print("\n\033[0;36m正在处理 deployment =>\033[0m")
        response = api_instance_deploy.list_namespaced_deployment(namespace)
        deploy_response = response.items
        deploy_size = len(deploy_response)

        '''加入进度条'''
        pbar1 = tqdm(total=deploy_size, unit='个', unit_scale=True)
        for i in deploy_response:
            deploy_team = i.metadata.labels.get('team')
            deploy_tag = i.metadata.labels.get('tag')
            deploy_cloud = i.metadata.labels.get('cloud')
            deploy_name = i.metadata.labels.get('app')
            deploy_yml = deploy_name + '.yml'

            # 创建对应目录
            teamdir = os.path.join(timedir, deploy_team)
            clouddir = os.path.join(teamdir, deploy_cloud)
            if not os.path.exists(clouddir):
                os.makedirs(clouddir)

            deploy_name_yml = os.path.join(clouddir, deploy_yml)  # 需存放路径不同，修改该行
            apiversion = i.metadata.annotations.get('kubectl.kubernetes.io/last-applied-configuration')

            all_dict = json.loads(apiversion)  # deployment 转换为字典

            '''组装，修改，更新部分区域'''
            # 删除不需要得部分
            all_dict['metadata'].pop('annotations')

            # 增加dns与host部分
            all_dict["spec"]["template"]["spec"]["hostAliases"] = k8s_host
            all_dict["spec"]["template"]["spec"]["dnsConfig"] = k8s_dns

            # 重新赋值镜像为当前
            deploy_image = i.spec.template.spec.containers[0].image
            all_dict['spec']['template']['spec']['containers'][0]['image'] = deploy_image
            # 重新赋值副本数replicas为当前
            deploy_scale = i.spec.replicas
            all_dict['spec']['replicas'] = deploy_scale
            # 修改拉取策略
            # deploy_policy = i.spec.template.spec.containers[0].image_pull_policy
            all_dict["spec"]["template"]["spec"]['containers'][0]['imagePullPolicy'] = 'IfNotPresent'

            # 修改livenessProbe
            if all_dict['spec']['template']['spec']['containers'][0].get('livenessProbe'):
                # 如果k8s内存中包含livenessProbe，首先将默认的5项赋值
                if i.spec.template.spec.containers[0].liveness_probe:
                    deploy_liveness_failure = i.spec.template.spec.containers[0].liveness_probe.failure_threshold
                    deploy_liveness_initial = i.spec.template.spec.containers[0].liveness_probe.initial_delay_seconds
                    deploy_liveness_period = i.spec.template.spec.containers[0].liveness_probe.period_seconds
                    deploy_liveness_success = i.spec.template.spec.containers[0].liveness_probe.success_threshold
                    deploy_liveness_timeout = i.spec.template.spec.containers[0].liveness_probe.timeout_seconds
                    # 重新赋值字典
                    liveness = all_dict['spec']['template']['spec']['containers'][0]['livenessProbe']
                    liveness['failureThreshold'] = deploy_liveness_failure
                    liveness['initialDelaySeconds'] = deploy_liveness_initial
                    liveness['periodSeconds'] = deploy_liveness_period
                    liveness['successThreshold'] = deploy_liveness_success
                    liveness['timeoutSeconds'] = deploy_liveness_timeout

                    # 判断如果字典里是tcp就重新赋值tcp，如果是http就清空tcp并重新赋值http
                    if liveness.get('tcpSocket'):
                        # 如果k8s内存中liveness中tcp是None，那么删除该tcp段，重新赋值http
                        if i.spec.template.spec.containers[0].liveness_probe.http_get:
                            # 设置字典中多余的tcpSocket值为空
                            liveness['tcpSocket'] = None
                            deploy_liveness_httppath = i.spec.template.spec.containers[0].liveness_probe.http_get.path
                            deploy_liveness_httpport = i.spec.template.spec.containers[0].liveness_probe.http_get.port
                            deploy_liveness_httpscheme = i.spec.template.spec.containers[
                                0].liveness_probe.http_get.scheme
                            liveness['httpGet'] = {}
                            liveness['httpGet']['path'] = deploy_liveness_httppath
                            liveness['httpGet']['port'] = deploy_liveness_httpport
                            liveness['httpGet']['scheme'] = deploy_liveness_httpscheme
                        if i.spec.template.spec.containers[0].liveness_probe.tcp_socket:
                            deploy_liveness_tcpport = i.spec.template.spec.containers[0].liveness_probe.tcp_socket.port
                            liveness['tcpSocket']['port'] = deploy_liveness_tcpport
                    all_dict['spec']['template']['spec']['containers'][0]['livenessProbe'] = liveness
                # print(all_dict['spec']['template']['spec']['containers'][0])

            # 修改readiness_probe
            if all_dict['spec']['template']['spec']['containers'][0].get('readinessProbe'):
                # 如果k8s内存中包含readiness_probe，首先将默认的5项赋值
                if i.spec.template.spec.containers[0].readiness_probe:
                    deploy_readiness_failure = i.spec.template.spec.containers[0].readiness_probe.failure_threshold
                    deploy_readiness_initial = i.spec.template.spec.containers[0].readiness_probe.initial_delay_seconds
                    deploy_readiness_period = i.spec.template.spec.containers[0].readiness_probe.period_seconds
                    deploy_readiness_success = i.spec.template.spec.containers[0].readiness_probe.success_threshold
                    deploy_readiness_timeout = i.spec.template.spec.containers[0].readiness_probe.timeout_seconds
                    # 重新赋值字典
                    readiness = all_dict['spec']['template']['spec']['containers'][0]['readinessProbe']
                    readiness['failureThreshold'] = deploy_readiness_failure
                    readiness['initialDelaySeconds'] = deploy_readiness_initial
                    readiness['periodSeconds'] = deploy_readiness_period
                    readiness['successThreshold'] = deploy_readiness_success
                    readiness['timeoutSeconds'] = deploy_readiness_timeout

                    # 判断如果字典里是tcp就重新赋值tcp，如果是http就清空tcp并重新赋值http
                    if readiness.get('tcpSocket'):
                        # 如果k8s内存中liveness中tcp是None，那么删除该tcp段，重新赋值http
                        if i.spec.template.spec.containers[0].readiness_probe.http_get:
                            # 设置字典中多余的tcpSocket值为空
                            readiness['tcpSocket'] = None
                            deploy_readiness_httppath = i.spec.template.spec.containers[0].readiness_probe.http_get.path
                            deploy_readiness_httpport = i.spec.template.spec.containers[0].readiness_probe.http_get.port
                            deploy_readiness_httpscheme = i.spec.template.spec.containers[
                                0].readiness_probe.http_get.scheme
                            readiness['httpGet'] = {}
                            readiness['httpGet']['path'] = deploy_readiness_httppath
                            readiness['httpGet']['port'] = deploy_readiness_httpport
                            readiness['httpGet']['scheme'] = deploy_readiness_httpscheme
                        if i.spec.template.spec.containers[0].readiness_probe.tcp_socket:
                            deploy_readiness_tcpport = i.spec.template.spec.containers[
                                0].readiness_probe.tcp_socket.port
                            readiness['tcpSocket']['port'] = deploy_readiness_tcpport
                    all_dict['spec']['template']['spec']['containers'][0]['readinessProbe'] = readiness
                # print(all_dict['spec']['template']['spec']['containers'][0]['readinessProbe'])

            # 修改ports
            if all_dict['spec']['template']['spec']['containers'][0].get('ports'):
                # 如果k8s内存中包含ports，修改默认
                if i.spec.template.spec.containers[0].ports:
                    deploy_port_container = i.spec.template.spec.containers[0].ports[0].container_port
                    deploy_port_protocol = i.spec.template.spec.containers[0].ports[0].protocol
                    # 重新赋值字典
                    ports = all_dict['spec']['template']['spec']['containers'][0]['ports'][0]
                    ports['containerPort'] = deploy_port_container
                    ports['protocol'] = deploy_port_protocol
                    all_dict['spec']['template']['spec']['containers'][0]['ports'][0] = ports

            '''利用字典将各个项目组的tag与各项目组下的cloud下的tag总结出来'''
            if data.get(deploy_team):
                data[deploy_team]['total'] = data.get(deploy_team).get('total') + 1
                data[deploy_team][deploy_tag] = data.get(deploy_team).get(deploy_tag) + 1
                if data.get(deploy_team).get(deploy_cloud):
                    if 'cloud' in deploy_cloud:
                        data[deploy_team][deploy_cloud]['total'] = data.get(deploy_team).get(deploy_cloud).get(
                            'total') + 1
                        data[deploy_team][deploy_cloud][deploy_tag] = data.get(deploy_team).get(deploy_cloud).get(
                            deploy_tag) + 1
                else:
                    data[deploy_team][deploy_cloud] = {'total': 1,
                                                       'grpc': 0,
                                                       'api': 0,
                                                       'mq': 0,
                                                       'scheduler': 0}
                    data[deploy_team][deploy_cloud][deploy_tag] = 1
            else:
                data[deploy_team] = {'total': 1,
                                     'grpc': 0,
                                     'api': 0,
                                     'mq': 0,
                                     'scheduler': 0}
                data[deploy_team][deploy_tag] = 1

            try:
                json1 = json.dumps(all_dict, separators=(',', ':'), ensure_ascii=False)  # 严格模式转换json
                ya = yaml.safe_load(json1)  # 转换为yaml格式

                with open(deploy_name_yml, 'w') as f1:
                    f1.write("# ------------------- Democloud Deployment ------------------- #\n")
                    yaml.safe_dump(ya, f1, default_flow_style=False)
                deploy_success_num += 1
            except:
                deploy_fail_num += 1
                deploy_fail_name.append(deploy_name)
            pbar1.update(1)

        pbar1.close()
        '''写个小的for循环来待会打印data字典中的key-即项目组team'''
        for k in data:
            team_list1.append(k)

        sleep(1)  # 停一秒等待一下，不然进度条显示有点问题
        print("\033[0;36m<= deployment 处理完成!\033[0m", "\n\033[0;32m 写入成功个数: ", deploy_success_num, "\033[0m",
              "\n\033[0;31m 写入失败个数: ", deploy_fail_num, " \033[0m")
        print("deployment操作中，共包含项目组：", len(data), "个", "分别为:", team_list1)
        print("xxx项目组cloud个数为:", len(data['xxx']) - 5, "\nyyy项目组cloud个数为:", len(data['yyy']) - 5,
              "\nzzz项目组cloud个数为:", len(data['zzz']) - 5, "\nqqq项目组cloud个数为:", len(data['qqq']) - 5)
        print("xxx项目组tag总数为:", data['xxx']['total'], "\nyyy项目组tag总数为:", data['yyy']['total'], "\nzzz项目组tag总数为:",
              data['zzz']['total'], "\nqqq项目组tag总数为:", data['qqq']['total'])
        if len(deploy_fail_name) > 0:
            print("失败的为:")
            for n in deploy_fail_name:
                print(n, end="\n")

    except ApiException as e:
        print("Exception when calling AppsV1Api->list_namespaced_deployment: %s\n" % e)


def service_all(namespace, timedir, data2):
    api_instance_service = client.CoreV1Api()  # list_namespaced_service Api接口不同
    service_success_num = 0
    service_fail_num = 0
    service_fail_name = []
    team_list2 = []
    try:
        print("\n\033[0;36m正在处理 service =>\033[0m")
        api_response = api_instance_service.list_namespaced_service(namespace)
        service_response = api_response.items
        service_size = len(service_response)
        '''加入进度条'''
        pbar2 = tqdm(total=service_size, unit='个', unit_scale=True)
        for j in service_response:
            content = j.metadata.annotations.get('kubectl.kubernetes.io/last-applied-configuration')
            service_team = j.metadata.labels.get('team')
            service_tag = j.metadata.labels.get('tag')
            service_cloud = j.metadata.labels.get('cloud')
            service_name = j.metadata.labels.get('app')

            content_dict = json.loads(content)  # service 转换为字典
            service_yml = service_name + ".yml"  # 获取server名,组合为.yml名

            # 组装对应目录
            teamdir2 = os.path.join(timedir, service_team)
            clouddir2 = os.path.join(teamdir2, service_cloud)
            if not os.path.exists(clouddir2):
                os.makedirs(clouddir2)

            service_name_yml = os.path.join(clouddir2, service_yml)  # 需存放路径不同，修改该行
            content_dict['metadata'].pop('annotations')  # 删除metadata中无用的annotations

            # 修改ports
            if content_dict['spec'].get('ports'):
                # 如果k8s内存中包含ports，修改默认
                if j.spec.ports:
                    service_port_p = j.spec.ports[0].port
                    service_port_target = j.spec.ports[0].target_port
                    service_port_protocol = j.spec.ports[0].protocol
                    # 重新赋值字典
                    service_ports = content_dict['spec']['ports'][0]
                    service_ports['port'] = service_port_p
                    service_ports['targetPort'] = service_port_target
                    service_ports['protocol'] = service_port_protocol
                    content_dict['spec']['ports'][0] = service_ports

            '''利用字典将各个项目组的tag与各项目组下的cloud下的tag总结出来'''
            if data2.get(service_team):
                data2[service_team]['total'] = data2.get(service_team).get('total') + 1
                data2[service_team][service_tag] = data2.get(service_team).get(service_tag) + 1
                if data2.get(service_team).get(service_cloud):
                    # 由于beatles cloud中包含一个scheduler，两个scheduler有冲突，故暂时取消计算该scheduler
                    if 'cloud' in service_cloud:
                        data2[service_team][service_cloud]['total'] = data2.get(service_team).get(service_cloud).get(
                            'total') + 1
                        data2[service_team][service_cloud][service_tag] = data2.get(service_team).get(
                            service_cloud).get(
                            service_tag) + 1
                else:
                    data2[service_team][service_cloud] = {'total': 1,
                                                          'grpc': 0,
                                                          'api': 0,
                                                          'mq': 0,
                                                          'scheduler': 0}
                    data2[service_team][service_cloud][service_tag] = 1
            else:
                data2[service_team] = {'total': 1,
                                       'grpc': 0,
                                       'api': 0,
                                       'mq': 0,
                                       'scheduler': 0}
                data2[service_team][service_tag] = 1

            try:
                json2 = json.dumps(content_dict, separators=(',', ':'), ensure_ascii=False)  # 严格模式转换json
                ya2 = yaml.safe_load(json2)  # 转换为yaml格式

                with open(service_name_yml, 'a') as f2:
                    f2.write("---\n# ------------------- Democloud Service ------------------- #\n")
                    yaml.safe_dump(ya2, f2, default_flow_style=False)
                service_success_num += 1
            except:
                service_fail_num += 1
                service_fail_name.append(service_name)

            pbar2.update(1)
        pbar2.close()
        for k in data2:
            team_list2.append(k)

        sleep(1)
        print("\033[0;36m<= service 处理完成!\033[0m", "\n\033[0;32m 写入成功个数: ", service_success_num, "\033[0m",
              "\n\033[0;31m 写入失败个数: ", service_fail_num, " \033[0m")
        print("service操作中，共包含项目组：", len(data2), "个", "分别为:", team_list2)
        print("xxx项目组cloud个数为:", len(data2['xxx']) - 5, "\nyyy项目组cloud个数为:", len(data2['yyy']) - 5,
              "\nzzz项目组cloud个数为:", len(data2['zzz']) - 5, "\nqqq项目组cloud个数为:", len(data2['qqq']) - 5)
        print("xxx项目组tag总数为:", data2['xxx']['total'], "\nyyy项目组tag总数为:", data2['yyy']['total'], "\nzzz项目组tag总数为:",
              data2['zzz']['total'], "\nqqq项目组tag总数为:", data2['qqq']['total'])
        if len(service_fail_name) > 0:
            print("失败的为:")
            for m in service_fail_name:
                print(m, end="\n")

    except ApiException as e:
        print("Exception when calling CoreV1Api->list_namespaced_service: %s\n" % e)


# 获取当天时间，并组成目录
def time_today(file_dir):
    timenow = datetime.date.today().strftime("%Y%m%d")
    timedir = os.path.join(file_dir, timenow)
    return timedir


# 获取7天以前的备份
def time():
    today = datetime.date.today()
    sevenday = datetime.timedelta(days=7)
    computing_time = today - sevenday
    old_day = computing_time.strftime("%Y%m%d")
    return old_day


# 删除7天以前的备份
def rm_olddir(file_dir, old_day):
    old_dir = os.path.join(file_dir, old_day)
    if os.path.exists(old_dir):
        shutil.rmtree(old_dir)


def main():
    # 获取并组建文件
    print("\033[0;34m程序开始运行处理  => \033[0m")
    kube_conf = os.path.join('F:\\', 'k8s', 'config')  # config存在路径
    file_dir = os.path.join('F:\\', 'k8s', 'yaml')  # 处理的yaml需保存目录，自行更改
    namespace = 'default'  # 需处理的命名空间,自行根据情况更改
    timedir = time_today(file_dir)
    data = {}  # 定义项目组
    data2 = {}
    config.load_kube_config(kube_conf)
    deployment_all(namespace, timedir, data)
    service_all(namespace, timedir, data2)
    print("\n\033[0;34m<= 所有文件均处理完成 !\033[0m")

    # 删除7天前的备份
    old_day = time()
    rm_olddir(file_dir, old_day)
    print("7天前备份删除完成，运行结束")


if __name__ == "__main__":
    main()

运行效果如下:

目录文件中见得到所有的deployment与service拼接yml文件

随便打开一个yml文件，格式没有问题

打完收工，感谢写脚本中，经理对我的帮助，由于目前对python不是很熟悉，不太熟悉，不了解的也太多