python 多进程/多线程/协程

import asyncio
import multiprocessing
from concurrent.futures.thread import ThreadPoolExecutor
from datetime import datetime
from time import sleep

import aiohttp
import requests


def request_url(n):
    rep = requests.get("https://www.baidu.com")
    return {"result": rep, "n": n}


def main_map():
    start_timestamp = datetime.now()
    thread_pool = ThreadPoolExecutor(max_workers=40, thread_name_prefix="test_")
    res = thread_pool.map(request_url, range(10000))
    thread_pool.shutdown(wait=True)

    for re in res:
        print(re)
    print("time cost：" + str(datetime.now() - start_timestamp))


def main_submit():
    start_timestamp = datetime.now()
    thread_pool = ThreadPoolExecutor(max_workers=40, thread_name_prefix="test_")

    task_list = []
    for i in range(20000):
        res = thread_pool.submit(request_url, i)
        task_list.append(res)

    thread_pool.shutdown()

    for re in task_list:
        print(re.result())
    print("time cost：" + str(datetime.now() - start_timestamp))


def _main_multiprocess():
    start_timestamp = datetime.now()
    print("CPU内核数:{}".format(multiprocessing.cpu_count()))
    pool = multiprocessing.Pool()

    task_list = []
    for i in range(2000):
        res = pool.apply_async(func=request_url, args=(i,))
        task_list.append(res)

    pool.close()
    pool.join()

    for re in task_list:
        print(re.get())
    print("time cost：" + str(datetime.now() - start_timestamp))


def request_url_thread_pool(n):
    start_timestamp = datetime.now()
    thread_pool = ThreadPoolExecutor(max_workers=40, thread_name_prefix="test_")

    task_list = []
    for i in range(n):
        res = thread_pool.submit(request_url, i)
        task_list.append(res)

    thread_pool.shutdown()

    for re in task_list:
        print(re.result())
    print("time cost：" + str(datetime.now() - start_timestamp))


def main_multiprocess(n):
    start_timestamp = datetime.now()
    print("CPU内核数:{}".format(multiprocessing.cpu_count()))
    count = multiprocessing.cpu_count()
    pool = multiprocessing.Pool()

    task_list = []
    for i in range(n):
        res = pool.apply_async(func=request_url_thread_pool, args=(int(n / count),))
        task_list.append(res)

    pool.close()
    pool.join()

    for re in task_list:
        print(re.get())
    print("time cost：" + str(datetime.now() - start_timestamp))


async def fetch_sub(i, semaphore):
    async with semaphore:
        async with aiohttp.ClientSession() as client:
            async with client.get("http://ad.partner.gifshow.com/track/activate?callback=zs-MVxyan0NeQX-GiQF2wBA1DfDUlBr9ylsXIFCdV24SOApPQb1-8YLgyiGg6jaEkEdxjtQuil7Z4vmJHtIsBnkUhS6AicDzki9LNOFEvkLNVI1qi8ximodInWFChyqj4c_oy0mOi0YRw_EwSjmAlviTniM5I51gJkie_U5e1AL-2XzvUjoQojNG5jzlAa&word=" + str(i)) as resp:
                return await resp.json()


if __name__ == '__main__':
    _main_multiprocess()
    # main_submit()
    # main_map()
    # main_multiprocess(20000)
    start_timestamp = datetime.now()
    loop = asyncio.get_event_loop()
    semaphore = asyncio.Semaphore(500)  # 限制并发量为500
    task_list = [asyncio.ensure_future(fetch_sub(i, semaphore)) for i in range(2000)]
    loop.run_until_complete(asyncio.wait(task_list))
    for re in task_list:
        print(re.result())
    print("time cost：" + str(datetime.now() - start_timestamp))