jenkins发布静态文件同时根据url即时刷新aliyun的cdn

jenkins的脚本

#!/bin/bash
#############################################
# 通过jenkins发布统一静态资源 neveragain.chinasoft.com 发布  注意/data/www/vhosts/neveragain.chinasoft.com/httpdocs/dist/ 发布到线上对应的是2019目录
# 
# 20200616 新增同时发布到国内一份，具体需求如下：
# DesignCenter.neveragain.chinasoft.com/dist/assets  这个目录下 icon,image,legacy,plugin,script,style,vendor 等目录中的图片，js，css等文件，发布到海外的同时发布一份到国内，并发布到CDN中
# 海外的域名是 neveragain.chinasoft.com  国内的域名是  allstatics.chinasoft.cn 

# 发布到海外 neveragain.chinasoft.com
web_iplist="1.1.1.1"

function neveragain_allstatics_ws_rsync()
{
for ip in $web_iplist
do
    echo "-- start pub --- 预发布到外网 ${ip} ----------------------------------------"
    /usr/local/bin/rsync -vazP --delete --bwlimit=1000 --exclude='.git/' --exclude='.gitignore/' --password-file=/data/www/.rsync/rsyncd.wsweb3 /data/www/vhosts/neveragain.chinasoft.com/httpdocs/dist/assets/ apache@${ip}::apache/data/www/vhosts/neveragain.chinasoft.com/httpdocs/2019/assets/
    if [[ $? == 0 || $? == 23 ]];then
            rsync_edit=1
    else
            rsync_edit=0
            echo "`date` rsync发布失败! -> editUrls.txt"
            exit 1
    fi

    echo -e "-- end pub ${ip} ----------------------------------------------------------\n\n"
done
}


# 发布到国内 allstatics.chinasoft.cn
alisz_publish_list="1.1.1.2"

function allstatics_alisz_rsync()
{
for ip in $alisz_publish_list
do
    echo "-- start pub --- 预发布到外网 ${ip} ----------------------------------------"
    /usr/local/bin/rsync -vazP --delete --bwlimit=1000 --exclude='.git/' --exclude='.gitignore/' --password-file=/data/www/.rsync/pass.allstatics.chinasoft.cn.sz_publish /data/www/vhosts/neveragain.chinasoft.com/httpdocs/dist/assets/ apache@${ip}::apache/data/www/vhosts/allstatics.chinasoft.cn/httpdocs/2019/assets/
    if [[ $? == 0 || $? == 23 ]];then
            rsync_edit=1
    else
            rsync_edit=0
            echo "`date` rsync发布失败! -> editUrls.txt"
            exit 1
    fi

    echo -e "-- end pub ${ip} ----------------------------------------------------------\n\n"
done
}

# 读取git的更新列表，发送请求调用python脚本刷新akamai CDN
function update_hw_cdn
{
    # 工作目录
    WORKSPACE="/data/jenkins_home/workspace/DesignCenter.neveragain.chinasoft.com/"
    cd $WORKSPACE

    # 获取变更列表
    changefiles=$(git diff --name-only HEAD~ HEAD)
    #echo $changefiles
    # 先把文件路径写死，作为测试使用
    #changefiles="dist/assets/image/box/drfone-mac.png dist/assets/image/box/drfone-win.png dist/assets/image/box/dvdcreator-mac.png dist/assets/image/box/dvdcreator-win.png dist/assets/image/box/famisafe.png dist/assets/image/box/filmora-mac.png dist/assets/image/box/filmora-win.png dist/assets/image/box/fotophire-editingtoolkit-mac.png dist/assets/image/box/fotophire-editingtoolkit-win.png dist/assets/image/box/fotophire-focus.png dist/assets/image/box/fotophire-maximizer.png dist/assets/image/box/fotophire-slideshowmaker-mac.png dist/assets/image/box/fotophire-slideshowmaker-win.png dist/assets/image/box/pdfelement-mac.png dist/assets/image/box/pdfelement-win.png dist/assets/image/box/recoverit-mac.png dist/assets/image/box/recoverit-win.png dist/assets/image/box/videoconverter-ultimate-mac.png dist/assets/image/box/videoconverter-ultimate-win.png dist/assets/image/box/chinasoft.png"

    #20190812103337
    now_time="`date +%Y%m%d%H%M%S`"
    # 将更新的文件列表写入日志文件中
    for newfile in $changefiles;
        do
                #echo $newfile
                start_str=${newfile:0:11}
                #dist/assets
                #echo $start_str
                # 如果变更的文件是 dist 目录下的文件就触发该文件刷新CDN
                if [ $start_str == 'dist/assets' ];then
                        need_file=${newfile:5}
                        #echo $need_file
                        need_url="https://neveragain.chinasoft.com/2019/$need_file"
                        #echo $need_url
                        echo "${need_url}" >> "/usr/local/worksh/jeninks_task/akamai_api/logs/${now_time}.log"
                fi
        done

# 调用Python脚本刷新cdn
/usr/local/worksh/jeninks_task/akamai_api_venv/bin/python /usr/local/worksh/jeninks_task/akamai_api/akamai_api.py $now_time
if [ $? != 0 ];then
    echo "刷新海外neveragain.chinasoft.com 的 akamai CDN失败"
    exit 1
else
    echo "刷新海外neveragain.chinasoft.com 的 akamai CDN成功"
fi

}

# 读取git的更新列表，发送请求调用python脚本刷新akamai CDN
function update_alisz_cdn
{
    # 工作目录
    WORKSPACE="/data/jenkins_home/workspace/DesignCenter.neveragain.chinasoft.com/"
    cd $WORKSPACE

    # 获取变更列表
    changefiles=$(git diff --name-only HEAD~ HEAD)
    #20190812103337
    now_time="`date +%Y%m%d%H%M%S`"
    # 将更新的文件列表写入日志文件中
    for newfile in $changefiles;
        do
                #echo $newfile
                start_str=${newfile:0:11}
                if [ $start_str == 'dist/assets' ];then
                        need_file=${newfile:5}
                        #echo $need_file
                        need_url="https://allstatics.chinasoft.cn/2019/$need_file"
                        #echo $need_url
                        echo "${need_url}" >> "/usr/local/worksh/jeninks_task/aliyun_cdn_api/logs/${now_time}.log"
                fi
        done

# 调用Python脚本刷新cdn
/usr/local/aliyun_cdn_api_venv/bin/python /usr/local/worksh/jeninks_task/aliyun_cdn_api/aliyun_cdn_api.py $now_time
if [ $? != 0 ];then
    echo "刷新阿里深圳allstatics.chinasoft.cn 的 akamai CDN失败"
    exit 1
else
    echo "刷新阿里深圳allstatics.chinasoft.cn 的 akamai CDN成功"
fi

}


def main()
{
    # 执行同步到海外网宿机房
    neveragain_allstatics_ws_rsync
    # 执行国内深圳机房www_web01和www_web02同步，jenkins --> publis_cms(jenkins调用apache用户下的脚本同步到) --> alisz_www_web01/alisz_www_web02
    allstatics_alisz_rsync
    # 刷新海外网宿为源站的akamai cdn
    update_hw_cdn
    # 刷新国内深圳机房www_web01和www_web02为源站的aliyun cdn
    update_alisz_cdn
}

# 入口函数
main

具体刷新aliyun cdn的脚本

# 安装软件包
# pip install aliyun-python-sdk-cdn aliyun-python-sdk-core requests urllib3
# /usr/local/worksh/jeninks_task/aliyun_cdn_api/aliyun_cdn_api.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
import requests, json,time,os,sys,datetime
import logging
import uuid,hmac,hashlib,base64
import urllib.parse

from aliyunsdkcore.client import AcsClient
from aliyunsdkcore.acs_exception.exceptions import ClientException
from aliyunsdkcore.acs_exception.exceptions import ServerException
from aliyunsdkcdn.request.v20180510.RefreshObjectCachesRequest import RefreshObjectCachesRequest
from aliyunsdkcdn.request.v20180510.PushObjectCacheRequest import PushObjectCacheRequest
from aliyunsdkcdn.request.v20180510.DescribeRefreshTasksRequest import DescribeRefreshTasksRequest
from aliyunsdkcdn.request.v20180510.DescribeRefreshQuotaRequest import DescribeRefreshQuotaRequest


#阿里刷新CDN的API
class aliyun_cdn_api(object):
    """
    阿里云的 API
    """
    def __init__(self):
        self.server = "https://cdn.aliyuncs.com"
        self.accessKeyId = "key"
        self.accessKeySecret = "pass"

    def percentEncode(self,arg_str):
        # res = urllib.quote(str(arg_str).decode(sys.stdin.encoding).encode('utf8'), '')
        res = urllib.parse.quote(str(arg_str).encode('utf8'), '')
        res = res.replace('+', '%20')
        res = res.replace('*', '%2A')
        res = res.replace('%7E', '~')
        return res

    def complate_singature(self,arg_map,accessKey):
        "生成Signature签名"
        # 对字段进行排序
        sort_args = sorted(arg_map.items(), key=lambda x: x[0])
        url_args_string = ''
        for k, v in sort_args:
            url_args_string += '&' + self.percentEncode(k) + '=' + self.percentEncode(v)
        stringToSign = 'GET&%2F&' + self.percentEncode(url_args_string[1:])
        #print(stringToSign)
        key_bytes= bytes(accessKey+ "&", 'utf-8')
        sign_bytes = bytes(stringToSign, 'utf-8')
        #h = hmac.new(accessKey + "&", stringToSign, hashlib.sha1)
        h = hmac.new(key_bytes, sign_bytes, hashlib.sha1)
        signature = base64.encodestring(h.digest()).strip()
        return signature

    def run(self,arg_map):
        args = {
            "Format": "JSON",
            "Version": "2018-05-10",
            "AccessKeyId": self.accessKeyId,
            "SignatureMethod": "HMAC-SHA1",
            "Timestamp": localtime_to_utctime(datetime.datetime.now()).strftime("%Y-%m-%dT%H:%M:%SZ"),       #UTC时间,格式为：YYYY-MM-DDThh:mm:ssZ
            "SignatureVersion": "1.0",
            "SignatureNonce": str(uuid.uuid1()),
        }
        args.update(arg_map)
        signature = self.complate_singature(args,self.accessKeySecret)
        args["Signature"] = signature
        try:
            response = self.getRequest(args)
        except Exception as err:
            return str(err)
        else:
            return response.json()

    def getRequest(self,data_map):
        retry = 0
        while retry < 10:
            try:
                response = requests.get(self.server,params=data_map)
            except Exception as err:
                retry += 1
                time.sleep(3)
                print("%s try again: %s" % (retry, str(err)))
                continue
            else:
                return response
        raise Exception("网络不可达,请检测服务器网络!!!")

    def RefreshCDNRequest(self,url_list):
        url_string = "\n".join(url_list)
        args = {
            "Action": "RefreshObjectCaches",
            "ObjectPath": url_string,
            "ObjectType" : "File",
        }
        return self.run(args)

    def DescribeRefreshCDNRequest(self,task_id):
        args = {
            "Action":"DescribeRefreshTasks",
            "TaskId": task_id,
        }
        return self.run(args)


def ReadFile(filename):
    """
    读取文件内容中的url路径
    每行一条url路径
    """
    l = []
    error_l = []
    with open(filename) as f:
        for url in  f.readlines():
            url_str = str(url).strip("\n")
            if str(url).startswith("https://allstatics.chinasoft.cn"):
                l.append(url_str)
            else:
                error_l.append(url_str)
    if error_l:
        raise Exception("The format of the file path is incorrect. %s"%('\n'.join(error_l)))
    return l

def localtime_to_utctime(local_datetime):
    """
    本地时间转UTC时间（-8:00）
    :param :
    :return:
    """
    time_struct = time.mktime(local_datetime.timetuple())
    utc_st = datetime.datetime.utcfromtimestamp(time_struct)
    return utc_st


if __name__ == "__main__":
    api = aliyun_cdn_api()
    #接收url文件名称
    if len(sys.argv) != 2:
        raise Exception("Not enough parameters for %s"%sys.argv[0])
    prefix_url_filename = sys.argv[1]
    # 定义日志级别
    baseDir = os.path.dirname(os.path.abspath(__file__))
    logfile = os.path.join(baseDir,"log.txt")
    logging.basicConfig(level=logging.INFO,
                        filename=logfile,
                        filemode='a',
                        format='%(asctime)s - %(filename)s - %(levelname)s: %(message)s')
    logger = logging.getLogger(__name__)

    #读取url的文件内容
    filename = os.path.join(baseDir,os.path.join("logs","%s.log"%prefix_url_filename))
    if not os.path.isfile(filename):
        raise Exception("Not exists file %s" %filename)
    url_list = ReadFile(filename)
    #print(url_list)

    #每次POST提交url的条数
    MAX_REQUEST_SIZE = 300
    while url_list:
        batch = []
        batch_size = 0

        while url_list and batch_size < MAX_REQUEST_SIZE:
            next_url = url_list.pop()
            batch.append(next_url)
            batch_size += 1
        if batch:
            response = api.RefreshCDNRequest(batch)
            #print(response)
            
            refresh_task_ids = response.get("RefreshTaskId","")
            if refresh_task_ids:
                retry = 0 #尝试10次
                success_map = {}    #用于保存多个RefreshTaskId的返回结果
                for refresh_task_id in str(refresh_task_ids).split(","):
                    res = api.DescribeRefreshCDNRequest(refresh_task_id)
                    while True:
                        res = api.DescribeRefreshCDNRequest(refresh_task_id)
                        print(res)
                        if 'Tasks' in res:
                            try:
                                state = res['Tasks']['CDNTask'][0]['Status']
                                if state ==  'Complete':
                                    success_map[refresh_task_id] = state
                                    break
                                elif state == 'Failed':
                                    success_map[refresh_task_id] = state
                                    is_success = False
                                    break
                                else:
                                    time.sleep(5)
                                    retry += 1
                                    logging.info("AliPurgeRequestTask 刷新 %s, 重试获取Ali的任务ID: %s" %(','.join(batch),refresh_task_id))
                                    continue
                                if retry > 10:
                                    success_map[refresh_task_id] = "请求发布CDN成功,但查询刷新状态失败."
                                    is_success = False
                                    #结束尝试
                                    break
                            except Exception as err:
                                success_map[refresh_task_id] = str(res)
                                is_success = False
                                break