为JenKins编写的自动打包脚本

前段时间为公司的一个项目部署了JenKins用于自动持续集成一个c++的项目。项目使用msbuild编译解决方案，在解决方案下包含了需要打包进最终的安装文件的可执行文件和部分静态库。在编译完解决方案后我想通过调用NSIS脚本直接创建一个安装包文件，于是就编写了一个python加到Jenkins Job的post build中。

脚本做的事情包括：

• 为每次build创建一个目录存放未打包的可执行文件。
• patch nsis脚本，以时间和SVN版本号为安装文件设定版本号和命名。
• 调用nsis打包文件并移动到指定的目录中（JenKins将这个目录下的安装包文件作为本次打包的成果）
• 保存所有符号文件(pdb)，按照visual stdio解决方案的目录，保存到一个指定的目录中。

由于使用的PyCharm edu对中文的支持不好，我在脚本中用简单的英文写了一些注释（不懂英文，里面的语法错误请忽略之）。

test_print_env 函数打印Jenkins的环境变量。

patch_nsis_script 函数修改NSIS脚本里的版本号和生成文件名。

save_symbols 函数保存符号到指定的目录

make_install 拷贝sln生成的可执行文件并且将它拷贝到为本次build创建的目录，调用NSIS打包和调用上面两个函数。

#!python.exe
# This script is writen for JenKins build shell command.
# This script will call the NSIS to pack all the XProj exe and DLL files.it work as follow :
# 1) Copy install files from $WORKSPACE\MakeInstall to \\MakeInstall\\app
# To use this sript:
# 1 make sure python 2.7.x has installed correctly.
# 2 please add the location of "makensis.exe" to system environment "PATH".#
# 3 set the system environment variable SYMBOL_SAVE_PATH to the path which save the ".pdb" file . if
#   it not set, default path is "D:\XProjSymbols"
# Note : U can translate lang to chs,but remember to save as UTF-8.
#
import os
import shutil
import datetime
import subprocess
import stat

def test_print_env():
    print("BUILD_TAG:", os.getenv("BUILD_TAG"))
    print("BRANCH_NAME:", os.getenv("BRANCH_NAME"))
    print("CHANGE_ID:", os.getenv("CHANGE_ID"))
    print("CHANGE_URL:", os.getenv("CHANGE_URL"))
    print("CHANGE_TITLE:", os.getenv("CHANGE_TITLE"))
    print("CHANGE_AUTHOR:", os.getenv("CHANGE_AUTHOR"))
    print("CHANGE_AUTHOR_DISPLAY_NAME:", os.getenv("CHANGE_AUTHOR_DISPLAY_NAME"))
    print("CHANGE_AUTHOR_EMAIL:", os.getenv("CHANGE_AUTHOR_EMAIL"))
    print("CHANGE_TARGET:", os.getenv("CHANGE_TARGET"))
    print("BUILD_NUMBER:", os.getenv("BUILD_NUMBER"))
    print("BUILD_ID:", os.getenv("BUILD_ID"))
    print("BUILD_DISPLAY_NAME:", os.getenv("BUILD_DISPLAY_NAME"))
    print("JOB_NAME:", os.getenv("JOB_NAME"))
    print("JOB_BASE_NAME:", os.getenv("JOB_BASE_NAME"))
    print("EXECUTOR_NUMBER:", os.getenv("EXECUTOR_NUMBER"))
    print("NODE_NAME:", os.getenv("NODE_NAME"))
    print("NODE_LABELS:", os.getenv("NODE_LABELS"))
    print("WORKSPACE:", os.getenv("WORKSPACE"))
    print("JENKINS_HOME:", os.getenv("JENKINS_HOME"))
    print("JOB_BASE_NAME:", os.getenv("JOB_BASE_NAME"))
    print("JENKINS_URL:", os.getenv("JENKINS_URL"))
    print("BUILD_URL:", os.getenv("BUILD_URL"))
    print("JOB_URL:", os.getenv("JOB_URL"))
    print("SVN_REVISION:", os.getenv("SVN_REVISION"))
    print("SVN_URL:", os.getenv("SVN_URL"))


def patch_nsis_script(jenkins_pack_root):
    make_install_file = "{0}\\MakeInstall\\xp_Install.nsi".format(jenkins_pack_root)
    if not os.path.exists(make_install_file):
        return None
    file_object = open(make_install_file, "r")
    all_the_text = file_object.read()
    file_object.close()
    # 1.Replace "SVN_VERSION_TAG" to ${SVN_REVISION}
    n_svn = os.getenv("SVN_REVISION")
    svn_tag = "SVN_VERSION_TAG"
    all_the_text = all_the_text.replace(svn_tag,n_svn)
    pos = all_the_text.find("!define PRODUCT_VERSION")
    if pos is None or pos == -1:
        print "Illegal file ,not found version string."
        return None
    else:
        text_len = len(all_the_text)
        begin_pos = None
        end_pos = None
        for i in range(pos + 1, text_len):
            char_in_text = all_the_text[i]
            if char_in_text == '\"' and begin_pos is not None:
                end_pos = i
                break
            elif char_in_text == "\"":
                begin_pos = i + 1
            else:
                pass
        old_version = all_the_text[begin_pos:end_pos]
        print "old version is :{0}".format(old_version)
        if begin_pos is None or end_pos is None:
            print "The version tag is not found."
            return None
        # Version string has found. patch it use current date.
        now_date_time = datetime.datetime.now()
        year_string = str(now_date_time.year)
        year_string = year_string[-2:]
        str_verion_number = "3.0.{0}.{1}{2}".format(year_string, now_date_time.month, now_date_time.day)
        print "new version is {0}".format(str_verion_number)
        if not old_version == str_verion_number:
            print "Replace version string with new version:"
            # Replace version strings
            new_script_text = all_the_text[0:begin_pos]
            # print new_script_text
            new_script_text = all_the_text[0:begin_pos] + str_verion_number + all_the_text[end_pos:]
            file_object = open(make_install_file, "w")
            file_object.write(new_script_text)
            file_object.close()
        else:
            print "Versions are same"

    return str_verion_number


def save_symbols(exts,src,dest):
    """
    Rebuild the director tree like src below dest and copy all files like XXX.exts to dest
    exts:exetens seperate by blank like "jpg png gif"
    """
    fp={}
    extss=exts.lower().split()
    for root,sub_dirs,files  in os.walk(src):
        for fl in files:
            if os.path.splitext(fl.lower())[1][1:] in extss:
                if root not in fp.keys():
                    fp[root]=[]
                fp[root].append(fl)
    for k,v in fp.items():
            relativepath=k[len(src)+1:]
            newpath=os.path.join(dest,relativepath)
            for f in v:
                oldfile=os.path.join(k,f)
                print("copy ["+oldfile+"] to ["+newpath+"]")
                if not os.path.exists(newpath):
                    os.makedirs(newpath)
                shutil.copy(oldfile,newpath)

# copy the files which need to packet to install script.
def make_install(is_debug):
    # n_build = "36"
    # n_svn = "543"
    # n_work_space = "F:\\JenkinsWorkSpace\\XProj"
    n_build = os.getenv("BUILD_NUMBER")
    n_svn = os.getenv("SVN_REVISION")
    n_temp = os.getenv("TEMP")
    n_work_space = os.getenv("WORKSPACE")
    if (n_build is not None) and (n_svn is not None) and (n_temp is not None) and (n_work_space is not None):
        # $name_pack assemble with build number and svn revision,with format as : "Build100Svn306"
        name_pack = "Build{0}Svn{1}".format(n_build, n_svn)
    else:
        raise Exception("Some env was not set correctly.check BUILD_NUMBER,SVN_REVISION,TEMP and WORKSPACE plz")
    # The path $temp_folder_pack is used to save binary file which need to pack by NSIS.
    # for example : "$WORKSPACE\JenkinsPack\Build100Svn306\"
    temp_folder_pack = os.path.join(n_work_space,"JenkinsPack")
    if not os.path.exists(temp_folder_pack):
        os.mkdir(temp_folder_pack)
    # The path $last_build is used for JenKins to store "Archive the artifacts" (JenKins Job setting).
    last_build = os.path.join(temp_folder_pack,"lastbuild")
    if not os.path.exists(last_build):
        os.mkdir(last_build)
    # for example : "$WORKSPACE\JenkinsPack\BuildxxxSvn3yyy\"
    temp_folder_pack = os.path.join(temp_folder_pack,name_pack)
    if os.path.exists(temp_folder_pack):
        raise Exception("{0} has already exist.".format(temp_folder_pack))
    os.mkdir(temp_folder_pack)
    if not os.path.exists(temp_folder_pack):
        raise Exception("make dir failed,{0} is not exist".format(temp_folder_pack))
    # copy files from  "$WORKSPACE\MakeInstall" to"$WORKSPACE\JenkinsPack\BuildxxxSvn3yyy\MakeInstall"
    copysrc = n_work_space + "\\MakeInstall"
    copydst = temp_folder_pack + "\\MakeInstall"
    print "Copy tree {0} to {1}".format(copysrc, copydst)
    shutil.copytree(copysrc, copydst)
    # Remove all sub folder with name ".svn" cause they were copied from local svn folder.
    # Otherwise they will be involved into the packet in NSIS building.
    for root,subdirs,files in os.walk(copydst):
        for filepath in files:
            print os.path.join(root,filepath)
        for sub in subdirs:
            fullpath =  os.path.join(root,sub)
            if sub == ".svn":
                # NOTE:shutil.rmtree() cannot delete readonly file, so we use os command for same purpose.
                os.chmod(fullpath, stat.S_IREAD | stat.S_IWRITE)
                os.system("rd /s /q {0}".format(fullpath))
    is_success = False
    # Copy project binary files made by MSBUILD.
    if True == is_debug:
        binary_file_list = {"\\bin\\xpUpgrade.exe": "\\MakeInstall\\app\\xpUpgrade.exe",
                            "\\bin\\Debug\\xpDefenseApi.dll": "\\MakeInstall\\app\\xpDefenseApi.dll",
                            "\\bin\\Debug\\xpLeakMainUI.exe": "\\MakeInstall\\app\\xpLeakMainUI.exe",
                            "\\bin\\Debug\\xpMain_Framework.exe": "\\MakeInstall\\app\\XProj.exe",
                            "\\bin\\Debug\\xpRuleSet.exe": "\\MakeInstall\\app\\xpRuleSet.exe",
                            "\\bin\\Debug\\xpSettings.exe": "\\MakeInstall\\app\\xpSettings.exe",
                            "\\bin\\Debug\\xpLeak.dll": "\\MakeInstall\\app\\xpLeak.dll",
                            "\\bin\\Debug\\xpSysFix.dll": "\\MakeInstall\\app\\xpSysFix.dll",
                            "\\Debug\\xpUpdate.dll": "\\MakeInstall\\app\\xpUpdate.dll",
                            "\\Debug\\xpSelfFix.exe": "\\MakeInstall\\app\\xpSelfFix.exe",
                            "\\Debug\\xpService.exe": "\\MakeInstall\\app\\xpService.exe",
                            "\\Debug\\xpTray.exe": "\\MakeInstall\\app\\xpTray.exe",
                            "\\Debug\\AmLogD.dll": "\\MakeInstall\\app\\AmLogD.dll",
                            "\\Debug\\xpAntivirus.dll": "\\MakeInstall\\app\\xpAntivirus.dll",
                            "\\Debug\\xpBrowserBase.dll": "\\MakeInstall\\app\\xpBrowserBase.dll",
                            "\\Debug\\xpBrowserSafe.dll": "\\MakeInstall\\app\\xpBrowserSafe.dll",
                            "\\Debug\\xpScanPath.dll": "\\MakeInstall\\app\\xpScanPath.dll",
                            "\\Debug\\xpTrashClean.dll": "\\MakeInstall\\app\\xpTrashClean.dll",
                            "\\xpOptimize\\bin\\Debug\\xpOptimize.dll":"\\MakeInstall\\app\\xpOptimize.dll",
                            "\\bin\\Debug\\xpAntiVirusUp.exe":"\\MakeInstall\\app\\xpAntiVirusUp.exe"
                            #"\\xpStatistics\\bin\\Debug\\xpOperation.dll": "\\MakeInstall\\app\\xpOperation.dll"
                            }
    else:
        binary_file_list = {"\\bin\\xpUpgrade.exe": "\\MakeInstall\\app\\xpUpgrade.exe",
                            "\\bin\\Release\\xpDefenseApi.dll": "\\MakeInstall\\app\\xpDefenseApi.dll",
                            "\\bin\\Release\\xpLeakMainUI.exe": "\\MakeInstall\\app\\xpLeakMainUI.exe",
                            "\\bin\\Release\\xpMain_Framework.exe": "\\MakeInstall\\app\\xpMain_Framework.exe",
                            "\\bin\\Release\\xpRuleSet.exe": "\\MakeInstall\\app\\xpRuleSet.exe",
                            "\\bin\\Release\\xpSettings.exe": "\\MakeInstall\\app\\xpSettings.exe",
                            "\\bin\\Release\\xpLeak.dll": "\\MakeInstall\\app\\xpLeak.dll",
                            "\\bin\\Release\\xpSysFix.dll": "\\MakeInstall\\app\\xpSysFix.dll",
                            "\\Release\\xpUpdate.dll": "\\MakeInstall\\app\\xpUpdate.dll",
                            "\\Release\\xpSelfFix.exe": "\\MakeInstall\\app\\xpSelfFix.exe",
                            "\\Release\\xpService.exe": "\\MakeInstall\\app\\xpService.exe",
                            "\\Release\\xpTray.exe": "\\MakeInstall\\app\\xpTray.exe",
                            "\\Release\\AmLog.dll": "\\MakeInstall\\app\\AmLog.dll",
                            "\\Release\\xpAntivirus.dll": "\\MakeInstall\\app\\xpAntivirus.dll",
                            "\\Release\\xpBrowserBase.dll": "\\MakeInstall\\app\\xpBrowserBase.dll",
                            "\\Release\\xpBrowserSafe.dll": "\\MakeInstall\\app\\xpBrowserSafe.dll",
                            "\\Release\\xpScanPath.dll": "\\MakeInstall\\app\\xpScanPath.dll",
                            "\\Release\\xpTrashClean.dll": "\\MakeInstall\\app\\xpTrashClean.dll",
                            #"\\xpStatistics\\bin\\Release\\xpOperation.dll": "\\MakeInstall\\app\\xpOperation.dll"
                            "\\xpOptimize\\bin\\Release\\xpOptimize.dll":"\\MakeInstall\\app\\xpOptimize.dll",
                            "\\bin\\Release\\xpAntiVirusUp.exe":"\\MakeInstall\\app\\xpAntiVirusUp.exe"
                            }

    for fromitem, toitem in binary_file_list.items():
        file_build = n_work_space + fromitem
        file_to_pack = temp_folder_pack + toitem
        print "copy {0} to {1}".format(file_build, file_to_pack)
        shutil.copy(file_build, file_to_pack)
    # We need to patch nsis script file to name the output file build by NSIS,with SVN revesion and build number.
    # The target file name is like "XProjWeiShiSetupV3.0.17.420(222)@innertest_CL.exe"
    str_new_ver = patch_nsis_script(temp_folder_pack)
    if str_new_ver is None:
        raise "Patch NSIS install script failed"
    filename = "XProjWeiShiSetupV{0}({1})@innertest_CL.exe".format(str_new_ver,n_svn)
    packed_file_name = "{0}\\MakeInstall\\bin\\{1}".format(temp_folder_pack, filename)
    # Call NSIS to make install.
    print "make nsis setup file : {0}".format(packed_file_name)
    strcmd = "makensis.exe"
    make_install_file = temp_folder_pack + "\\MakeInstall\\xp_Install.nsi"
    strcmd = strcmd + " " + make_install_file
    print strcmd
    result = subprocess.call(strcmd)
    print result
    # Check the target file exist or not.
    if not os.path.exists(packed_file_name):
        raise Exception("Error!the file {0} which suppose to make install was not found!".format(packed_file_name))
    print "File {0} make success".format(packed_file_name)
    # delete all files in last_build folder
    for root,subdirs,files in os.walk(last_build):
        for filepath in files:
            fullpath = os.path.join(root,filepath)
            os.system("del /f /q {0}".format(fullpath)) # NOTE:shutil.rmtree() cannot delete readonly file.
            print "remove file : {0}".format(fullpath)
        for sub in subdirs:
            fullpath =  os.path.join(root,sub)
            os.chmod(fullpath, stat.S_IREAD | stat.S_IWRITE)
            os.system("rd /s /q {0}".format(fullpath)) # NOTE:shutil.rmtree() cannot delete readonly file.
            print "remove dir : {0}".format(fullpath)
    # Copy file to $last_build folder
    last_build = os.path.join(last_build,filename) # "{0}\\{1}".format(last_build,filename)
    shutil.copy(packed_file_name, last_build)
    # Read symbol path from environment with name "SYMBOL_SAVE_PATH".
    # If not exist ,we just save it to "D:\XProjSymbols\$name_pack"
    n_save_path = os.getenv("SYMBOL_SAVE_PATH")
    if n_save_path is None:
        n_save_path = "D:\\XProjSymbols"
    if not os.path.exists(n_save_path):
        os.mkdir(n_save_path)
    n_save_path = os.path.join(n_save_path,name_pack)
    if not os.path.exists(n_save_path):
        os.mkdir(n_save_path)
    save_symbols("pdb",n_work_space,n_save_path)
    is_success = True
    return


if __name__ == '__main__':
    print("Python script for JenKins.")
    test_print_env()
    make_install(True)