使用paramiko 进行多线程本地代码远端打包，及本地打总包

# -*- coding: utf-8 -*-
#! /usr/bin/python
import paramiko
import os
import time
import threading
import commands
import sys

master_running = True
agent_running = True


class SSHConnection(object):
    def __init__(self, host_dict):
        self.host = host_dict['host']
        self.port = host_dict['port']
        self.username = host_dict['username']
        self.passwd = host_dict['passwd']
        self.__k = None
 
    def connect(self):
        transport = paramiko.Transport((self.host, self.port))
        transport.connect(username=self.username,password=self.passwd)
        self.__transport = transport
 
    def close(self):
        self.__transport.close()
 
    def run_cmd(self, command):
        ssh = paramiko.SSHClient()
        ssh._transport = self.__transport
        stdin, stdout, stderr = ssh.exec_command(command)
        # 获取命令结果
        res = to_str(stdout.read())
        # 获取错误信息
        error = to_str(stderr.read())
        # 如果有错误信息，返回error
        # 否则返回res
        if error.strip():
            return False, error
        else:
            return True, res
 
    def upload(self, local_path, target_path, exclude_args):
        def put_file(lo_path, ta_path):
            if not os.path.exists(lo_path):
                print "%s is not exist" % lo_path
                exit()
            if os.path.isdir(lo_path):
                self.run_cmd('mkdir %s' % ta_path)
                file_list = os.listdir(lo_path)
                for file in file_list:
                    if file in exclude_args:
                        continue
                    new_lo = lo_path + '/' + file
                    new_ta = ta_path + '/' + file
                    put_file(new_lo, new_ta)
            else:
                # 连接，上传
                sftp = paramiko.SFTPClient.from_transport(self.__transport)
                # 将location.py 上传至服务器 /tmp/test.py
                sftp.put(lo_path, ta_path, confirm=True)
                # print lo_path
                # print(os.stat(local_path).st_mode)
                # 增加权限
                # sftp.chmod(target_path, os.stat(local_path).st_mode)
                # sftp.chmod(target_path, 0o755)  # 注意这里的权限是八进制的，八进制需要使用0o作为前缀
        put_file(local_path, target_path)

    def download(self, target_path, local_path):
        def get_file(ta_path, lo_path):
            if os.path.isdir(ta_path):
                os.system('mkdir %s' % lo_path)
                file_list = os.listdir(ta_path)
                for file in file_list:
                    new_lo = lo_path + '/' + file
                    new_ta = ta_path + '/' + file
                    get_file(new_ta, new_lo)
            else:
                # 连接，上传
                sftp = paramiko.SFTPClient.from_transport(self.__transport)
                # 将location.py 上传至服务器 /tmp/test.py
                sftp.get(ta_path, lo_path)
                print ta_path
        get_file(target_path, local_path)
 
    def __del__(self):
        self.close()


def to_str(bytes_or_str):
    """
    把byte类型转换为str
    :param bytes_or_str:
    :return:
    """
    if isinstance(bytes_or_str, bytes):
        value = bytes_or_str.decode('utf-8')
    else:
        value = bytes_or_str
    return value


def master_work(server_conf, excludes):
    # conf set
    master_dir = "xdfs-restful/xdfs-mgnt-master/src"
    remote_master_dir = "/root/rpmbuild/SOURCES/xdfs-mgnt-master-tcmu-5.5.6/src"
    remote_package_master = "/root/master-package-5.5.6-tcmu"
    remote_rpm_master = "/root/rpmbuild/RPMS/noarch/xdfs-mgnt-master-tcmu-5.5.6-1.el7.noarch.rpm"
    # connection
    ssh = SSHConnection(server_conf)
    ssh.connect()
    if not os.path.exists("%s" % master_dir):
        print "has not dir %s" % master_dir
        exit()
    ssh.run_cmd("cd %s && rm -rf *" % remote_master_dir)
    # make master rpm
    ssh.upload(master_dir, remote_master_dir, excludes)
    print "-" * 80 + "finish master code"
    ssh.run_cmd("sh %s" % remote_package_master)
    print "-" * 80 + "finish master rpm"
    # download rpm
    filename_master = remote_rpm_master.split('/')[-1]
    ssh.download(remote_rpm_master, filename_master)
    ssh.__del__()
    global master_running
    master_running = False


def agent_work(server_conf, excludes):
    # config set
    agent_dir = "xdfs-restful/xdfs-mgnt-agent/src"
    remote_agent_dir = "/root/rpmbuild/SOURCES/xdfs-mgnt-agent-tcmu-5.5.6/src"
    remote_package_agent = "/root/agent-package-5.5.6-tcmu"
    remote_rpm_agent = "/root/rpmbuild/RPMS/noarch/xdfs-mgnt-agent-tcmu-5.5.6-1.el7.noarch.rpm"
    # connection
    ssh = SSHConnection(server_conf)
    ssh.connect()
    check_s3 = os.listdir('xdfs-restful/xdfs-mgnt-agent/src/agent/xdfs_s3')
    if len(check_s3) == 0:
        print "-" * 80 + "need git clone xdfs_s3"
    if not os.path.exists("%s" % agent_dir):
        print "has not dir %s" % agent_dir
        exit()
    # delete remote code
    ssh.run_cmd("cd %s && rm -rf *" % remote_agent_dir)
    # make agent rpm
    ssh.upload(agent_dir, remote_agent_dir, excludes)
    print "-" * 80 + "finish agent code"
    ssh.run_cmd("sh %s" % remote_package_agent)
    print "-" * 80 + "finish agent rpm"
    # download rpm
    filename_agent = remote_rpm_agent.split('/')[-1]
    ssh.download(remote_rpm_agent, filename_agent)
    ssh.__del__()
    global agent_running
    agent_running = False


def tar_package():
    os.system('pv %s | tar -zxf -' % sys.argv[1])
    ret = os.listdir('.')
    if sys.argv[3] not in ret:
        print "file(%s) is not exist" % sys.argv[3]
        exit()

    # conf master and agent rpm
    rpm_dir = '{0}/02_MGNT'.format(sys.argv[3])
    master_rpm = '{0}/02_MGNT/xdfs-mgnt-master*.rpm'.format(sys.argv[3])
    agent_rpm = '{0}/02_MGNT/xdfs-mgnt-agent*.rpm'.format(sys.argv[3])


    time_start = int(time.time())
    global agent_running, master_running
    while True:
        if not agent_running and not master_running:
            break
        time_end = int(time.time())
        if time_end - time_start > 600:
            break
    # select to delete, add
    ret_xdfs = os.listdir('.')
    for item in ret_xdfs:
        if 'xdfs-mgnt-agent' in item and '.rpm' in item:
            os.system('rm -rf %s' % agent_rpm)
            os.system('mv xdfs-mgnt-agent*.rpm %s' % rpm_dir)
        if 'xdfs-mgnt-master' in item and '.rpm' in item:
            os.system('rm -rf %s' % master_rpm)
            os.system('mv xdfs-mgnt-master*.rpm %s' % rpm_dir)

    os.system('tar -zcf %s %s |pv -t' % (sys.argv[2], sys.argv[3]))
    os.system('rm -rf %s' % sys.argv[3])
    print "-" * 80 + "finish tar"


if __name__ == '__main__':
    # python remote_mk_tar.py XDFS-5.5.6-0218.tar.gz XDFS-5.5.6-0219.tar.gz XDFS-5.5.6-IMGHA_Install
    ret = os.listdir('.')
    if len(sys.argv) == 4:
        if sys.argv[1] not in ret:
            print "file(%s) is not exist" % sys.argv[1]
            exit()
    # server config
    server_dict = {
        "host": "172.16.105.1",
        "port": 22,
        "username": "root",
        "passwd": "111111",
    }
    # check rpmbuild
    ssh = SSHConnection(server_dict)
    ssh.connect()
    ret = ssh.run_cmd("ls")
    if not ret[0]:
        print ret[1]
        exit()
    if "rpmbuild" not in ret[1]:
        print "remote_server need install rpmbuild, and mkdir rpmbuild"
        exit()
    # exclude master dir or file
    master_exclude = ['ems']
    # exclude agent dir or file
    agent_exclude = []
    master_thread = threading.Thread(args=(server_dict, master_exclude), target=master_work)
    agent_thread = threading.Thread(args=(server_dict, agent_exclude), target=agent_work)
    tar_thread = threading.Thread(args=(), target=tar_package)
    #master_running = False
    master_thread.start()
    #agent_running = False
    agent_thread.start()
    tar_thread.start()