批量操作安装功能

功能结构设计：

   分别使用单线程、多线程、多进程、协程分别实现功能：

一、 单线程实现功能

1.1单线程 实现发送点击生成 transaction_id;

click01.py  ：  发送点击模块；

base.py:                          封装一些公共的方法；

# _*_ encoding=utf-8 _*_
import requests,re
from common.test_datas import tt
from modules.base import BaseModule
from modules.executors import thread_pool_executor as tp


class Click(BaseModule):

    def __init__(self):
        super(Click,self).__init__()

    def _process(self, offer_id):
        host = '```'
        click_id = ''.join(re.findall('[0-9a-z]', (tt.radom_uuid())))
        url = host + ":8888/click?id=" + offer_id + "&aff=31110005&click_id=" + click_id + "&ip=89.179.45.132&pkg=pkg12345&lang=lang&pkg=com.test.cn&ua=ua&ad_type=ad_type&media_type=media_type&model=model&os=Android&sub_siteid=123213&offer_name=offernameselenium2022-07-05&android_id=35433&os_version=os_version&brand=brand&b_size=b_size&date=20220706&data_source_id=1645619743316148226"
        res = requests.get(url=url, verify=False)
        if "Success=false" in res.text:
            print("结束程序运行 {}".format(res.text))
        else:
            transaction_id = ((res.url).split("transaction_id=")[1]).split("&affiliate_id=")[0]
        return transaction_id

    def _process_singlethread(self,list_):
        # 单线程运行
        response_list = [self._process(offer) for offer in list_]
        return response_list

1.2单线程 实现操作安装;

doingPB02.py     单线程实现安装功能；

# _*_ encoding=utf-8 _*_
import requests
from modules.base import BaseModule
from modules.executors import thread_pool_executor as tp

class DoingPB(BaseModule):

    def _process(self,transaction_id):
        host = '  ···   '
        url = host + ":8090/pb/lsr?click_id=" + transaction_id
        print("点击系统执行PB操作 【transaction_id：{},请求url: {}】".format(transaction_id, url))
        res = requests.get(url=url, verify=False)
        return res.text


    def _process_singlethread(self,list_):
        # 单线程运行
        response_list = [self._process(transaction_id) for transaction_id in list_]
        return response_list

1.3单线程  实现调度;

schedulerinstall.py   单线程实现调度功能；

# _*_ encoding=utf-8 _*_
import os.path
import time

from modules.click01 import Click
from modules.doingPB02 import DoingPB

from const import CalcType
class SchedulerInstall:

    def __init__(self):
        self.ck = Click()
        self.doingpb = DoingPB()


    def process(self):
        # 1.加载要执行的offerid
        lit = ['1006','538']

        # 2. 调度发送点击
        content_list = self.ck.process(lit)
        # 3. 调度操作安装
        self.doingpb.process(content_list)

if __name__ == '__main__':
    scheduleri = SchedulerInstall()
    scheduleri.process()

二、 多线程实现功能

2.1多线程 实现发送点击生成 transaction_id;

click01.py  ：  发送点击模块；

# _*_ encoding=utf-8 _*_
import requests,re
from common.test_datas import tt
from modules.base import BaseModule
from modules.executors import thread_pool_executor as tp


class Click(BaseModule):

    def __init__(self):
        super(Click,self).__init__()

    def _process(self, offer_id):
        host = '  ···   '
        click_id = ''.join(re.findall('[0-9a-z]', (tt.radom_uuid())))
        url = host + ":8888/click?id=" + offer_id + "&aff=31110005&click_id=" + click_id + "&ip=89.179.45.132&pkg=pkg12345&lang=lang&pkg=com.test.cn&ua=ua&ad_type=ad_type&media_type=media_type&model=model&os=Android&sub_siteid=123213&offer_name=offernameselenium2022-07-05&android_id=35433&os_version=os_version&brand=brand&b_size=b_size&date=20220706&data_source_id=1645619743316148226"
        # res = requests.get(url=url, headers=header, verify=False)
        res = requests.get(url=url, verify=False)
        if "Success=false" in res.text:
            print("结束程序运行 {}".format(res.text))
        else:
            transaction_id = ((res.url).split("transaction_id=")[1]).split("&affiliate_id=")[0]
        return transaction_id

    

    def _process_multithread(self,list_):
        # 多线程 下载
        task_list = [tp.submit(self._process,(offer)) for offer in list_]
        response_list = [task.result() for task in task_list]
        return response_list

2.2多线程 实现操作安装;

doingPB02.py     多线程实现安装功能；

# _*_ encoding=utf-8 _*_
import requests
from modules.base import BaseModule
from modules.executors import thread_pool_executor as tp

class DoingPB(BaseModule):

    def _process(self,transaction_id):
        host = ' ···   '
        url = host + ":8090/pb/lsr?click_id=" + transaction_id
        print("点击系统执行PB操作 【transaction_id：{},请求url: {}】".format(transaction_id, url))
        res = requests.get(url=url, verify=False)
        return res.text


    def _process_multithread(self,list_):
        # 多线程 运行
        task_list = []
        for transaction_id in list_:
            task = tp.submit(self._process, (transaction_id))
            task_list.append(task)
        for task in task_list:
            task.result()

2.3多线程  实现调度;

schedulerinstall.py   多线程实现调度功能；

# _*_ encoding=utf-8 _*_
from modules.click01 import Click
from modules.doingPB02 import DoingPB

from const import CalcType
class SchedulerInstall:

    def __init__(self):
        self.ck = Click()
        self.doingpb = DoingPB()
        self.ck.set_calc_type(CalcType.MultiThread)
        self.doingpb.set_calc_type(CalcType.MultiThread)


    def process(self):
        # 1.加载要执行的offerid
        lit = ['1006','538']

        # 2. 调度发送点击
        content_list = self.ck.process(lit)
        # 3. 调度操作安装
        self.doingpb.process(content_list)

if __name__ == '__main__':
    scheduleri = SchedulerInstall()
    scheduleri.process()

三、多进程实现功能

3.1多进程 实现发送点击生成 transaction_id;

click01.py  ：  发送点击模块；

# _*_ encoding=utf-8 _*_
import requests,re
from common.test_datas import tt
from modules.base import BaseModule
from modules.executors import thread_pool_executor as tp
from modules.executors import process_pool_executor as pp


class Click(BaseModule):

    def __init__(self):
        super(Click,self).__init__()

    def _process(self, offer_id):
        host = '  ```   '
        click_id = ''.join(re.findall('[0-9a-z]', (tt.radom_uuid())))
        url = host + ":8888/click?id=" + offer_id + "&aff=31110005&click_id=" + click_id + "&ip=89.179.45.132&pkg=pkg12345&lang=lang&pkg=com.test.cn&ua=ua&ad_type=ad_type&media_type=media_type&model=model&os=Android&sub_siteid=123213&offer_name=offernameselenium2022-07-05&android_id=35433&os_version=os_version&brand=brand&b_size=b_size&date=20220706&data_source_id=1645619743316148226"
        # res = requests.get(url=url, headers=header, verify=False)
        res = requests.get(url=url, verify=False)
        if "Success=false" in res.text:
            print("结束程序运行 {}".format(res.text))
        else:
            transaction_id = ((res.url).split("transaction_id=")[1]).split("&affiliate_id=")[0]
        return transaction_id

    def _process_multiprocess(self, list_):
        # 多进程 下载
        response_list = []

        task_list = []
        for offer in list_:
            task = pp.submit(self._process, (offer))
            task_list.append(task)
        for task in task_list:
            transaction_id = task.result()
            response_list.append(transaction_id)
        return response_list

3.2多进程 实现操作安装;

doingPB02.py     多进程实现安装功能；

# _*_ encoding=utf-8 _*_
import requests
from modules.base import BaseModule
from modules.executors import thread_pool_executor as tp
from modules.executors import process_pool_executor as pp

class DoingPB(BaseModule):

    def _process(self,transaction_id):
        host = '  ···   '
        url = host + ":8090/pb/lsr?click_id=" + transaction_id
        print("点击系统执行PB操作 【transaction_id：{},请求url: {}】".format(transaction_id, url))
        res = requests.get(url=url, verify=False)
        return res.text


    def _process_multiprocess(self, list_):
        # 多进程 下载
        task_list = []
        for transaction_id in list_:
            task = pp.submit(self._process, (transaction_id))
            task_list.append(task)
        for task in task_list:
            task.result()

3.3多进程  实现调度;

schedulerinstall.py   多进程实现调度功能；

# _*_ encoding=utf-8 _*_
from modules.click01 import Click
from modules.doingPB02 import DoingPB
import utils,prettytable
from const import CalcType
class SchedulerInstall:

    def __init__(self):
        self.ck = Click()
        self.doingpb = DoingPB()
        self.ck.set_calc_type(CalcType.MultiProcess)
        self.doingpb.set_calc_type(CalcType.MultiProcess)


    def process(self):
        time_statictics = {}
        time_statictics['click_time'] = []
        time_statictics['install_time'] = []
        timer = utils.Timer()

        # 1.加载要执行的offerid
        lit = ['1006','538']

        # 2. 调度发送点击
        timer.tick()
        content_list = self.ck.process(lit)
        time_cost = timer.tock()
        time_statictics['click_time'].append(time_cost)
        # 3. 调度操作安装
        timer.tick()
        self.doingpb.process(content_list)
        time_cost = timer.tock()
        time_statictics['install_time'].append(time_cost)

        return time_statictics


if __name__ == '__main__':
    scheduleri = SchedulerInstall()
    log_time = scheduleri.process()
    

3.4分别对单线程，多线程，多进程 实现调度;

分别对单线程，多线程，多进程执行代码汇总，并对执行时间绘制表格输出；

# _*_ encoding=utf-8 _*_
from modules.click01 import Click
from modules.doingPB02 import DoingPB
import utils,prettytable
from const import CalcType
class SchedulerInstall:

    def __init__(self):
        self.ck = Click()
        self.doingpb = DoingPB()


    def set_calc_type(self,type_):
        self.ck.set_calc_type(type_)
        self.doingpb.set_calc_type(type_)


    def process(self):
        time_statictics = {}
        time_statictics['click_time'] = []
        time_statictics['install_time'] = []
        timer = utils.Timer()

        # 1.加载要执行的offerid
        lit = ['1006','538']

        # 2. 调度发送点击
        timer.tick()
        content_list = self.ck.process(lit)
        time_cost = timer.tock()
        time_statictics['click_time'].append(time_cost)
        # 3. 调度操作安装
        timer.tick()
        self.doingpb.process(content_list)
        time_cost = timer.tock()
        time_statictics['install_time'].append(time_cost)

        return time_statictics

    def statictics(self, single_log, multi_log, multiprocess_log):
        table = prettytable.PrettyTable(['类型', '单线程总耗时','多线程耗时','多线程提升率','多进程耗时','多进程提升率'])
        click_row = ['click']
        install_row = ['install']

        click_row.append(single_log['click_time'][0])
        install_row.append(single_log['install_time'][0])

        #  多线程数据
        click_row.append(multi_log['click_time'][0])
        install_row.append(multi_log['install_time'][0])
        # 多线程提升率
        time_ = single_log['click_time'][0] - multi_log['click_time'][0]
        lift_rate = '%.4f%%' % ((time_ / single_log['click_time'][0]) * 100)
        click_row.append(lift_rate)

        time_ = single_log['install_time'][0] - multi_log['install_time'][0]
        lift_rate = '%.4f%%' % ((time_ / single_log['install_time'][0]) * 100)
        install_row.append(lift_rate)

        #  多进程数据
        click_row.append(multiprocess_log['click_time'][0])
        install_row.append(multiprocess_log['install_time'][0])

        # 多进程提升率
        time_ = single_log['click_time'][0] - multiprocess_log['click_time'][0]
        lift_rate = '%.4f%%' % ((time_ / single_log['click_time'][0]) * 100)
        click_row.append(lift_rate)

        time_ = single_log['install_time'][0] - multiprocess_log['install_time'][0]
        lift_rate = '%.4f%%' % ((time_ / single_log['install_time'][0]) * 100)
        install_row.append(lift_rate)


        table.add_row(click_row)
        table.add_row(install_row)
        print(table)


if __name__ == '__main__':
    scheduleri = SchedulerInstall()
    # 单线程运行
    scheduleri.set_calc_type(CalcType.SingleThread)
    singlethread_time = scheduleri.process()

    # 多线程运行
    scheduleri.set_calc_type(CalcType.MultiThread)
    multithread_time = scheduleri.process()

    # 多进程运行
    scheduleri.set_calc_type(CalcType.MultiProcess)
    multiprocess_time = scheduleri.process()

    # 执行时间合并，分析
    scheduleri.statictics(singlethread_time, multithread_time, multiprocess_time)

|   类型  | 单线程总耗时 | 多线程耗时 | 多线程提升率 | 多进程耗时 | 多进程提升率 |
+---------+--------------+------------+--------------+------------+--------------+
|  click  |   1.923541   |  1.144035  |   40.5245%   |   1.7682   |   8.0758%    |
| install |   2.605678   |  1.359892  |   47.8104%   |  1.418436  |   45.5636%   |
+---------+--------------+------------+--------------+------------+--------------+

四、协程实现功能

4.1协程 实现发送点击生成 transaction_id;

click01.py  ：  发送点击模块；

# _*_ encoding=utf-8 _*_
import requests,re
from common.test_datas import tt
from modules.base import BaseModule
from modules.executors import thread_pool_executor as tp
from modules.executors import process_pool_executor as pp
from modules.executors import pycoroutine_executor as pe
import httpx,asyncio

host = ' ··· '

class Click(BaseModule):

    def __init__(self):
        super(Click,self).__init__()

    def _process(self, offer_id):
        click_id = ''.join(re.findall('[0-9a-z]', (tt.radom_uuid())))
        url = host + ":8888/click?id=" + offer_id + "&aff=31110005&click_id=" + click_id + "&ip=89.179.45.132&pkg=pkg12345&lang=lang&pkg=com.test.cn&ua=ua&ad_type=ad_type&media_type=media_type&model=model&os=Android&sub_siteid=123213&offer_name=offernameselenium2022-07-05&android_id=35433&os_version=os_version&brand=brand&b_size=b_size&date=20220706&data_source_id=1645619743316148226"
        res = requests.get(url=url, verify=False)
        if "Success=false" in res.text:
            print("结束程序运行 {}".format(res.text))
        else:
            transaction_id = ((res.url).split("transaction_id=")[1]).split("&affiliate_id=")[0]
        return transaction_id

        def _process_singlethread(self,list_):
        # 单线程运行
        response_list = [self._process(offer) for offer in list_]
        return response_list

    def _process_multithread(self,list_):
        # 多线程 下载
        task_list = [tp.submit(self._process,(offer)) for offer in list_]
        response_list = [task.result() for task in task_list]
        return response_list

    def _process_multiprocess(self, list_):
        # 多进程 下载
        task_list = [pp.submit(self._process, (offer)) for offer in list_]
        response_list = [task.result() for task in task_list]
        return response_list

    def _process_coroutine(self, list_):

        click_id = ''.join(re.findall('[0-9a-z]', (tt.radom_uuid())))
        response_list = []
        async def main():
            for offer in list_:
                url = host + ":8888/click?id=" + offer + "&aff=31110005&click_id=" + click_id + "&ip=89.179.45.132&pkg=pkg12345&lang=lang&pkg=com.test.cn&ua=ua&ad_type=ad_type&media_type=media_type&model=model&os=Android&sub_siteid=123213&offer_name=offernameselenium2022-07-05&android_id=35433&os_version=os_version&brand=brand&b_size=b_size&date=20220706&data_source_id=1645619743316148226"
                async with httpx.AsyncClient() as response:
                    response = await response.get(url)
                    res = response.headers['location']
                    #
                    transaction_id = (res.split("transaction_id=")[1]).split("&affiliate_id=")[0]
                    response_list.append(transaction_id)

        pe.run_until_complete(main())
        return response_list

4.2协程 实现操作安装;

doingPB02.py     协程实现安装功能；

# _*_ encoding=utf-8 _*_
import requests,httpx
from modules.base import BaseModule
from modules.executors import thread_pool_executor as tp
from modules.executors import process_pool_executor as pp
from modules.executors import pycoroutine_executor as pe
host = '  ···   '
class DoingPB(BaseModule):

    def _process(self,transaction_id):

        url = host + ":8090/pb/lsr?click_id=" + transaction_id
        print("点击系统执行PB操作 【transaction_id：{},请求url: {}】".format(transaction_id, url))
        res = requests.get(url=url, verify=False)
        return res.text


        def _process_singlethread(self,list_):
        # 单线程运行
        response_list = [self._process(transaction_id) for transaction_id in list_]
        return response_list

    def _process_multithread(self,list_):
        # 多线程 运行
        response_list = []
        task_list = [tp.submit(self._process, (transaction_id)) for transaction_id in list_]
        for task in task_list:
            response = task.result()
            response_list.append(response)
        return response_list

    def _process_multiprocess(self, list_):
        # 多进程 下载
        response_list = []
        task_list = [pp.submit(self._process, (transaction_id)) for transaction_id in list_]
        for task in task_list:
            response = task.result()
            response_list.append(response)
        return response_list

    def _process_coroutine(self, list_):
        # 协程
        return self._process_multiprocess(list_)

4.3协程  实现调度;

schedulerinstall.py   协程实现调度功能；

# _*_ encoding=utf-8 _*_
from modules.click01 import Click
from modules.doingPB02 import DoingPB
import utils,prettytable
from const import CalcType
class SchedulerInstall:

    def __init__(self):
        self.ck = Click()
        self.doingpb = DoingPB()


    def set_calc_type(self,type_):
        self.ck.set_calc_type(type_)
        self.doingpb.set_calc_type(type_)


    def process(self):
        time_statictics = {}
        time_statictics['click_time'] = []
        time_statictics['install_time'] = []
        timer = utils.Timer()

        # 1.加载要执行的offerid
        lit = ['1006', '538']

        # 2. 调度发送点击
        timer.tick()
        content_list = self.ck.process(lit)
        time_cost = timer.tock()
        time_statictics['click_time'].append(time_cost)
        # 3. 调度操作安装
        timer.tick()
        self.doingpb.process(content_list)
        time_cost = timer.tock()
        time_statictics['install_time'].append(time_cost)

        return time_statictics

    def statictics(self, single_log, multi_log, multiprocess_log, pycoroutine_log):
        table = prettytable.PrettyTable(['类型', '单线程总耗时','多线程耗时','多线程提升率','多进程耗时','多进程提升率','协程','协程提升率'])
        click_row = ['click']
        install_row = ['install']

        click_row.append(single_log['click_time'][0])
        install_row.append(single_log['install_time'][0])

        #  多线程数据
        click_row.append(multi_log['click_time'][0])
        install_row.append(multi_log['install_time'][0])
        # 多线程提升率
        time_ = single_log['click_time'][0] - multi_log['click_time'][0]
        lift_rate = '%.4f%%' % ((time_ / single_log['click_time'][0]) * 100)
        click_row.append(lift_rate)

        time_ = single_log['install_time'][0] - multi_log['install_time'][0]
        lift_rate = '%.4f%%' % ((time_ / single_log['install_time'][0]) * 100)
        install_row.append(lift_rate)

        #  多进程数据
        click_row.append(multiprocess_log['click_time'][0])
        install_row.append(multiprocess_log['install_time'][0])

        # 多进程提升率
        time_ = single_log['click_time'][0] - multiprocess_log['click_time'][0]
        lift_rate = '%.4f%%' % ((time_ / single_log['click_time'][0]) * 100)
        click_row.append(lift_rate)

        time_ = single_log['install_time'][0] - multiprocess_log['install_time'][0]
        lift_rate = '%.4f%%' % ((time_ / single_log['install_time'][0]) * 100)
        install_row.append(lift_rate)

        #  协程数据
        click_row.append(pycoroutine_log['click_time'][0])
        install_row.append(pycoroutine_log['install_time'][0])

        # 协程提升率
        time_ = single_log['click_time'][0] - pycoroutine_log['click_time'][0]
        lift_rate = '%.4f%%' % ((time_ / single_log['click_time'][0]) * 100)
        click_row.append(lift_rate)

        time_ = single_log['install_time'][0] - pycoroutine_log['install_time'][0]
        lift_rate = '%.4f%%' % ((time_ / single_log['install_time'][0]) * 100)
        install_row.append(lift_rate)


        table.add_row(click_row)
        table.add_row(install_row)
        print(table)


if __name__ == '__main__':
    scheduleri = SchedulerInstall()
    # 单线程运行
    scheduleri.set_calc_type(CalcType.SingleThread)
    singlethread_time = scheduleri.process()

    # 多线程运行
    scheduleri.set_calc_type(CalcType.MultiThread)
    multithread_time = scheduleri.process()

    # 多进程运行
    scheduleri.set_calc_type(CalcType.MultiProcess)
    multiprocess_time = scheduleri.process()

    # 多进程运行
    scheduleri.set_calc_type(CalcType.PyCoroutine)
    pyprocess_time = scheduleri.process()

    # 执行时间合并，分析
    scheduleri.statictics(singlethread_time, multithread_time, multiprocess_time,pyprocess_time)

实际运行结果：

+---------+--------------+------------+--------------+------------+--------------+----------+------------+
|   类型 | 单线程总耗时| 多线程耗时| 多线程提升率| 多进程耗时|多进程提升率| 协程 | 协程提升率 |
+---------+--------------+------------+--------------+------------+--------------+----------+------------+
|  click  |   3.005457 |  1.566382|   47.8821% |  2.299507 |   23.4889%  | 1.495633 |  50.2361%  |
| install |   3.423995 |  1.787206|   47.8035% |  1.726279 |   49.5829%  | 1.738925 |  49.2136%  |
+---------+--------------+------------+--------------+------------+--------------+----------+------------+