asyncio + aiohttp协程异步并发示例

1、简单示例

import asyncio
from aiohttp import ClientSession,TCPConnector
import time

async def aiohttp_get(page):
    url = 'https://www.baidu.com/s?wd={}'.format(page)
    try:
        conn = TCPConnector(limit=10)  # 限制同时链接数，连接默认是100，limit=0 无限制
        async with ClientSession(connector=conn) as session:
                async with session.get(url,timeout=3) as response:
                    if response.status==200:
                        html = await response.text()
                        return {'url':url,'html':html}
                    else:
                        return {'url':url,'http_status':response.status}
    except Exception as err:
        print('Request error:',err)
        return None

def parse(task):
    print('返回值：',task.result())

async def main():
    tasks = []
    for i in range(10):
        task = asyncio.create_task(aiohttp_get(i))
        task.add_done_callback(parse)  # 异步获取返回值并输出给回调函数处理
        tasks.append(task)
    await asyncio.gather(*tasks)

if __name__ == '__main__':
    start = time.time()
    loop = asyncio.get_event_loop()
    loop.run_until_complete(main())
    print(time.time()-start)
    print('程序结束')

2、设置回调函数参数

from functools import partial

def parse2(page,task):
    print('Page:',page,',返回值:',task.result())

async def main():
    tasks = []
    for i in range(10):
        task = asyncio.create_task(aiohttp_get(i))
        task.add_done_callback(partial(parse2,i))  # 设置回调函数参数
        tasks.append(task)
    await asyncio.gather(*tasks)

3、等待所有返回值后再输出

async def main():
    tasks = [aiohttp_get(i) for i in range(10)]
    results = await asyncio.gather(*tasks)
    print('结果已全部返回！')
    for result in results:
        print(len(result))

4、输出结果