使用python实现服务器下载功能
#!/usr/bin/env python3
#!coding:utf-8
# author : huangchengwu
# describe : 用于接收新资源下载后进行上传obj
import gevent
from gevent import monkey
monkey.patch_all()
import os
import sys
import time
import requests
from obs import ObsClient
import shutil
from collections import namedtuple
from ansible.parsing.dataloader import DataLoader
from ansible.vars.manager import VariableManager
from ansible.inventory.manager import InventoryManager
from ansible.playbook.play import Play
from ansible.executor.task_queue_manager import TaskQueueManager
from ansible.executor.playbook_executor import PlaybookExecutor
from ansible.plugins.callback import CallbackBase
import ansible.constants as C
import json
#ansible回调函数 所有ansible输出都在这 重写ansible输出方法进行输出
class ResultCallback(CallbackBase):
"""
重构输出
"""
def __init__(self, *args, **kwargs):
super(ResultCallback, self).__init__(*args, **kwargs)
self.host_ok = {}
self.host_unreachable = {}
self.host_failed = {}
def v2_runner_on_ok(self, result, **kwargs):
"""成功"""
self.host_ok[result._host.name] = result._result["stdout"]
def v2_runner_on_unreachable(self, result, **kwargs):
"""不可达"""
self.host_unreachable[result._host.name] = result._result["msg"]
def v2_runner_on_failed(self, result, ignore_errors=False, **kwargs):
"""失败"""
self.host_failed[result._host.name] = result._result["stderr"]
class AnsibleApi():
def __init__(self):
#初始化我抄的 试了很多遍这个配置最好使
self.Options = namedtuple('Options',
['connection',
'remote_user',
'ask_sudo_pass',
'verbosity',
'ack_pass',
'module_path',
'forks',
'become',
'become_method',
'become_user',
'check',
'listhosts',
'listtasks',
'listtags',
'syntax',
'sudo_user',
'sudo',
'diff'])
self.ops = self.Options(connection='smart',
remote_user=None,
ack_pass=None,
sudo_user=None,
forks=5,
sudo=None,
ask_sudo_pass=False,
verbosity=5,
module_path=None,
become=None,
become_method=None,
become_user=None,
check=False,
diff=False,
listhosts=None,
listtasks=None,
listtags=None,
syntax=None)
# 主要加载设置的变量
self.loader = DataLoader()
# 一个密码参数，可以设置为None，默认即可，没什么影响，我用的是秘钥登录
self.passwords = dict()
# dict(vault_pass='secret')
# 结果回调
self.callback = ResultCallback()
def runner(self, hosts, ansible_host_path, module, args):
# 设置传入的机器清单
inventory = InventoryManager(
loader=self.loader, sources=[ansible_host_path])
# 加载之前的变量
variable_manager = VariableManager(
loader=self.loader, inventory=inventory)
play_source = dict(
name="Ansible Play",
hosts=hosts, # all表示匹配清单所有机器，看源码发现的
gather_facts="no",
tasks=[
dict(action=dict(module=module, args=args), register='shell_out')
]
)
play = Play().load(play_source, variable_manager=variable_manager, loader=self.loader)
tqm = None
try:
tqm = TaskQueueManager(
inventory=inventory,
variable_manager=variable_manager,
loader=self.loader,
options=self.ops,
passwords=self.passwords,
stdout_callback=self.callback,
)
result = tqm.run(play)
finally:
if tqm is not None:
tqm.cleanup()
shutil.rmtree(C.DEFAULT_LOCAL_TMP, True)
# 重构输出 重构输出到json
result_raw = {'success': {}, 'failed': {}, 'unreachable': {}}
for host, result in self.callback.host_ok.items():
result_raw["success"][host] = result
for host, result in self.callback.host_unreachable.items():
result_raw['failed'][host] = result
for host, result in self.callback.host_failed.items():
result_raw['unreachable'][host] = result
#这只一下json的换行4
return json.dumps(result_raw, indent=4)
#ansible剧本在这 传入方法 a.runner_playbook("./init.yaml", "./host.ini")
def runner_playbook(self, playbook_path, ansible_host_path):
inventory = InventoryManager(
loader=self.loader, sources=[ansible_host_path])
variable_manager = VariableManager(
loader=self.loader, inventory=inventory)
pbex = PlaybookExecutor(playbooks=[playbook_path],
inventory=inventory,
variable_manager=variable_manager,
loader=self.loader,
options=self.ops,
passwords=self.passwords)
results = pbex.run()
print(results)
#传入如下 支持ip 支持组名
#a.runner(["192.168.1.207", "192.168.1.208"],
# "./host.ini", "shell", "echo 1")
# a.runner("test",
# "./host.ini", "shell", "echo 1")
def Arunner(self, ansible_host_path, module, args):
inventory = InventoryManager(
loader=self.loader, sources=[ansible_host_path])
# 加载之前的变量
variable_manager = VariableManager(
loader=self.loader, inventory=inventory)
play_source = dict(
name="Ansible Play",
hosts="all", # all表示匹配清单所有机器，看源码发现的
gather_facts="no",
tasks=[
dict(action=dict(module=module, args=args), register='shell_out'),
]
)
play = Play().load(play_source, variable_manager=variable_manager, loader=self.loader)
tqm = None
try:
tqm = TaskQueueManager(
inventory=inventory,
variable_manager=variable_manager,
loader=self.loader,
options=self.ops,
passwords=self.passwords,
stdout_callback=self.callback,
)
result = tqm.run(play)
finally:
if tqm is not None:
tqm.cleanup()
shutil.rmtree(C.DEFAULT_LOCAL_TMP, True)
# 重构输出
result_raw = {'success': {}, 'failed': {}, 'unreachable': {}}
for host, result in self.callback.host_ok.items():
result_raw["success"][host] = result
for host, result in self.callback.host_unreachable.items():
result_raw['failed'][host] = result
for host, result in self.callback.host_failed.items():
result_raw['unreachable'][host] = result
return json.dumps(result_raw, indent=4)
class Objtools():
#初始化配置包括obs key skey
def __init__(
self,
access_key_id="9DSDLW5TSCUBU0XHOLG4",
secret_access_key="HB4vH98psY3gvmfZaKDZFs1MyITK8z5jnLKnujkC",
objurl="https://obs.cn-north-4.myhuaweicloud.com",
urlfile="./url.txt",
cdnmap="./cdnMapping.json",
logdir="./log/",
downloaddir="./download",
):
AnsibleApi.__init__(self)
self.key = access_key_id
self.skey = secret_access_key
self.server = objurl
self.urlfile = urlfile
self.cdnmap = cdnmap
self.logdir = logdir
self.downloaddir = downloaddir
self.downloadtask = []
self.obsClient = ObsClient(
access_key_id=access_key_id,
secret_access_key=secret_access_key,
server=objurl
)
# 文件夹初始化
def initmkdir(self):
if os.path.exists(self.downloaddir) == False:
print("文件不存在创建目录")
os.mkdir(self.downloaddir)
if os.path.exists(self.logdir) == False:
print("文件不存在创建目录")
os.mkdir(self.logdir)
#打开文件夹返回句柄
def openfile(self, name, mode):
try:
f = open(name, mode, encoding='utf-8')
except expression as identifier:
print(name, mode, "打开文件失败")
return f
# 写入日志
def logwrite(self, content):
f = self.openfile(self.logdir+"/Objtools.log", "a+")
localtime = time.asctime(time.localtime(time.time()))
f.write("{}------{}------\n".format(localtime, content))
f.close()
# http方式下载
def download(self, url, file_path):
try:
#stram 流方式发开 HTTP1.1后支持 和pipline管道很像
r1 = requests.get(url, stream=True, verify=False)
#请求文件得知他的大小判断本地字节 来做一个断点续传
total_size = int(r1.headers['Content-Length'])
#请求资源不存在则返回不会下载
if r1.status_code != 200:
print("请求错误",url,file_path,r1.status_code,"\n")
return
except:
print("请求失败",url)
print("下载失败",file_path)
return
# 这重要了，先看看本地文件下载了多少
if os.path.exists(file_path):
temp_size = os.path.getsize(file_path) # 本地已经下载的文件大小
else:
temp_size = 0
# 显示一下下载了多少
print(temp_size)
print(total_size)
#传给服务器一个范围 服务器会把你只想某一块来拿数据
# 核心部分，这个是请求下载时，从本地文件已经下载过的后面下载
headers = {'Range': 'bytes=%d-' % temp_size}
# 重新请求网址，加入新的请求头的
r = requests.get(url, stream=True, verify=False, headers=headers)
# "ab"表示追加形式写入文件
with open(file_path, "ab") as f:
#文件加锁 保证这个文件只有一个线程在写 其他想写都要等这个线程写完
fcntl.flock(f, fcntl.LOCK_EX)
for chunk in r.iter_content(chunk_size=100000):
if chunk:
temp_size += len(chunk)
f.write(chunk)
f.flush()
###这是下载实现进度显示####
done = int(50 * temp_size / total_size)
sys.stdout.write("\r[%s%s] %d%%" % (
'█' * done, ' ' * (50 - done), 100 * temp_size / total_size))
sys.stdout.flush()
#解锁关闭文件
fcntl.flock(f, fcntl.LOCK_UN)
f.close()
# 获取objs列表
def getobjlist(self, bucketName, objectKey):
destfile = {}
self.logwrite("获取obj列表")
resp = self.obsClient.listObjects(bucketName, objectKey)
if resp.status < 300:
self.logwrite("请求成功 {} {} ".format(bucketName, objectKey))
try:
for i in resp.body.contents:
d = str(i).split("/")
l = len(d)-1
if d[l].strip() != "":
destfile[d[l]] = True
except expression as identifier:
self.logwrite("读取obj列表失败")
else:
self.logwrite('请求失败', bucketName, objectKey)
return destfile
#下载 拿obs桶中的文件对比哪些该不该下 并且拿url和名字进行一个多协程下载
def DoRun(self, destfile):
f = self.openfile(self.urlfile, "rt")
#二次去重
_f = set(f.readlines())
for i in _f:
l = i.split("http:")
n = len(l)-1
if l[n] == "\n" or l[n] == "":
continue
url = "http:"+l[n]
_n = url.split("/")
filename = _n[len(_n)-1].replace("\n", "").strip()
if destfile.get(filename) == None:
print("没有就开始下载", url.strip(), filename)
self.downloadtask.append(gevent.spawn(
self.download, url.strip(), self.downloaddir+"/"+filename.strip()))
#开启协程传入一个列表
gevent.joinall(self.downloadtask)
# 上传
def upload(self, destfile):
f = self.openfile(self.urlfile, "rt")
_f = set(f.readlines())
for i in _f:
l = i.split("http:")
n = len(l)-1
if l[n] == "\n" or l[n] == "":
continue
url = "http:"+l[n]
_n = url.split("/")
filename = _n[len(_n)-1].replace("\n", "").strip()
if destfile.get(filename) == None:
try:
r1 = requests.get(url, stream=True, verify=False)
if r1.status_code == 200:
total_size = int(r1.headers['Content-Length'])
temp_size = os.path.getsize(self.downloaddir+"/"+filename) # 本地已经下载的文件大小
print(total_size ,temp_size)
if temp_size >= total_size:
print("已下载完成",filename)
#去查询obs是否有这个文件
#没有上传 上传后执行更新配置文件
#有不上传 不上传不更新
else:
print("未完成下载",filename)
except:
print("检测文件失败",url,r1.status_code)
def upcdnmap(self):
#获取上传内容并且更新配置
#ansible同步到各文件
# a = AnsibleApi()
# data = a.Arunner("./host.ini", "shell", "echo 1")
# ansible_host_path = os.path.join(os.getcwd(), "hosts")
# print("=------------", ansible_host_path)
# data = a.runner(["192.168.1.207", "192.168.1.208"],
# "./host.ini", "shell", "echo 1")
# data = a.runner("test",
# "./host.ini", "shell", "echo 1")
# print(data)
# ansible-playbook
# a.runner_playbook("./init.yaml", "./host.ini")
print("修改cdnmap")
if __name__ == "__main__":
o = Objtools()
o.initmkdir()
filearry = o.getobjlist("obs-61c73", "pkg")
o.DoRun(filearry)
o.upload(filearry)
o.upcdnmap()
ansible
#!/usr/bin/env python
# 1、python3.6和ansible2.7.8 api环境
# 2、封装ansible2.7.8 api
# author huangchengwu
import os
import json
import shutil
from collections import namedtuple
from ansible.parsing.dataloader import DataLoader
from ansible.vars.manager import VariableManager
from ansible.inventory.manager import InventoryManager
from ansible.playbook.play import Play
from ansible.executor.playbook_executor import PlaybookExecutor
from ansible.executor.task_queue_manager import TaskQueueManager
from ansible.plugins.callback import CallbackBase
import ansible.constants as C
class ResultCallback(CallbackBase):
"""
重构输出
"""
def __init__(self, *args, **kwargs):
super(ResultCallback, self).__init__(*args, **kwargs)
self.host_ok = {}
self.host_unreachable = {}
self.host_failed = {}
def v2_runner_on_ok(self, result, **kwargs):
"""成功"""
self.host_ok[result._host.name] = result._result["stdout"]
def v2_runner_on_unreachable(self, result, **kwargs):
"""不可达"""
self.host_unreachable[result._host.name] = result._result["msg"]
def v2_runner_on_failed(self, result, ignore_errors=False, **kwargs):
"""失败"""
self.host_failed[result._host.name] = result._result["stderr"]
class AnsibleApi():
def __init__(self):
# self.Options = namedtuple('Options',
# ['connection',
# 'module_path',
# 'forks',
# 'private_key_file',
# 'remote_user',
# 'become',
# 'become_method',
# 'become_user',
# 'check',
# 'diff'])
# self.ops = self.Options(connection='smart',
# module_path=None,
# forks=10,
# private_key_file="/root/.ssh/id_rsa", # 你的私钥
# remote_user="root", # 远程用户
# become=True,
# become_method="sudo",
# become_user="root",
# check=False,
# diff=False)
self.Options = namedtuple('Options',
['connection',
'remote_user',
'ask_sudo_pass',
'verbosity',
'ack_pass',
'module_path',
'forks',
'become',
'become_method',
'become_user',
'check',
'listhosts',
'listtasks',
'listtags',
'syntax',
'sudo_user',
'sudo',
'diff'])
self.ops = self.Options(connection='smart',
remote_user=None,
ack_pass=None,
sudo_user=None,
forks=5,
sudo=None,
ask_sudo_pass=False,
verbosity=5,
module_path=None,
become=None,
become_method=None,
become_user=None,
check=False,
diff=False,
listhosts=None,
listtasks=None,
listtags=None,
syntax=None)
# 主要加载设置的变量
self.loader = DataLoader()
# 一个密码参数，可以设置为None，默认即可，没什么影响，我用的是秘钥登录
self.passwords = dict()
# dict(vault_pass='secret')
# 结果回调
self.callback = ResultCallback()
def runner(self, hosts, ansible_host_path, module, args):
# 设置传入的机器清单
inventory = InventoryManager(
loader=self.loader, sources=[ansible_host_path])
# 加载之前的变量
variable_manager = VariableManager(
loader=self.loader, inventory=inventory)
play_source = dict(
name="Ansible Play",
hosts=hosts, # all表示匹配清单所有机器，看源码发现的
gather_facts="no",
tasks=[
dict(action=dict(module=module, args=args), register='shell_out')
]
)
play = Play().load(play_source, variable_manager=variable_manager, loader=self.loader)
tqm = None
try:
tqm = TaskQueueManager(
inventory=inventory,
variable_manager=variable_manager,
loader=self.loader,
options=self.ops,
passwords=self.passwords,
stdout_callback=self.callback,
)
result = tqm.run(play)
finally:
if tqm is not None:
tqm.cleanup()
shutil.rmtree(C.DEFAULT_LOCAL_TMP, True)
# 重构输出
result_raw = {'success': {}, 'failed': {}, 'unreachable': {}}
for host, result in self.callback.host_ok.items():
result_raw["success"][host] = result
for host, result in self.callback.host_unreachable.items():
result_raw['failed'][host] = result
for host, result in self.callback.host_failed.items():
result_raw['unreachable'][host] = result
return json.dumps(result_raw, indent=4)
def runner_playbook(self, playbook_path, ansible_host_path):
inventory = InventoryManager(
loader=self.loader, sources=[ansible_host_path])
variable_manager = VariableManager(
loader=self.loader, inventory=inventory)
pbex = PlaybookExecutor(playbooks=[playbook_path],
inventory=inventory,
variable_manager=variable_manager,
loader=self.loader,
options=self.ops,
passwords=self.passwords)
results = pbex.run()
print(results)
def Arunner(self, ansible_host_path, module, args):
inventory = InventoryManager(
loader=self.loader, sources=[ansible_host_path])
# 加载之前的变量
variable_manager = VariableManager(
loader=self.loader, inventory=inventory)
play_source = dict(
name="Ansible Play",
hosts="all", # all表示匹配清单所有机器，看源码发现的
gather_facts="no",
tasks=[
dict(action=dict(module=module, args=args), register='shell_out'),
]
)
play = Play().load(play_source, variable_manager=variable_manager, loader=self.loader)
tqm = None
try:
tqm = TaskQueueManager(
inventory=inventory,
variable_manager=variable_manager,
loader=self.loader,
options=self.ops,
passwords=self.passwords,
stdout_callback=self.callback,
)
result = tqm.run(play)
finally:
if tqm is not None:
tqm.cleanup()
shutil.rmtree(C.DEFAULT_LOCAL_TMP, True)
# 重构输出
result_raw = {'success': {}, 'failed': {}, 'unreachable': {}}
for host, result in self.callback.host_ok.items():
result_raw["success"][host] = result
for host, result in self.callback.host_unreachable.items():
result_raw['failed'][host] = result
for host, result in self.callback.host_failed.items():
result_raw['unreachable'][host] = result
return json.dumps(result_raw, indent=4)
if __name__ == "__main__":
a = AnsibleApi()
# data = a.Arunner("./host.ini", "shell", "echo 1")
# ansible_host_path = os.path.join(os.getcwd(), "hosts")
# print("=------------", ansible_host_path)
# data = a.runner("test", "./host.ini", "shell", "echo 1")
# data1 = a.runner("test", "./host.ini", "shell", "echo 2")
# print(data)
a.runner_playbook("./init.yaml", "./host.ini")