python 异步编程——asyncio

摘要

本文主要介绍了python通过asyncio模块如何实现异步编程。

1. 协程

1.1 基本概念

1. 协程不是计算机提供的，程序员人为创造；
2. 协程（coroutine，微线程），即一个线程实现代码块相互切换执行；

1.2 实现方法

1. greenlet模块（早期）
2. yield关键字
3. asyncio模块（py3.4）
4. async、await关键字（py3.5）

1.2.1 greenlet

pip install greenlet

from greenlet import greenlet
def func_a():
    print("first in func_a!")
    gr_b.switch()
    print("second in func_a!")
    gr_b.switch()
    print("third in func_a!")
    gr_b.switch()
    
def func_b():
    print("first in func_b!")
    gr_a.switch()
    print("second in func_b!")
    gr_a.switch()
    print("third in func_b!")

gr_a = greenlet(func_a)
gr_b = greenlet(func_b)

gr_a.switch()

first in func_a!
first in func_b!
second in func_a!
second in func_b!
third in func_a!
third in func_b!

1.2.2 yield

def func_a():
    yield "first in func_a!"
    yield from func_b()
    yield "second in func_a!"
    yield "third in func_a!"

def func_b():
    yield "first in func_b!"
    yield "second in func_b!"
    yield "third in func_b!"
iters = func_a()
for i in iters:
    print(i)

first in func_a!
first in func_b!
second in func_b!
third in func_b!
second in func_a!
third in func_a!

1.2.3 asyncio模块

import asyncio


@asyncio.coroutine
def func_a():
    print("first in func_a!")
    yield from asyncio.sleep(2)
    print("second in func_a!")
    yield from asyncio.sleep(2)
    print("third in func_a!")


@asyncio.coroutine
def func_b():
    print("first in func_b!")
    yield from asyncio.sleep(2)
    print("second in func_b!")
    yield from asyncio.sleep(2)
    print("third in func_b!")


tasks = [asyncio.ensure_future(func_a()),
         asyncio.ensure_future(func_b())]
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.wait(tasks))

first in func_a!
first in func_b!
second in func_a!
second in func_b!
third in func_a!
third in func_b!

1.2.4 async、await关键字（推荐）

import asyncio



async def func_a():
    print("first in func_a!")
    await asyncio.sleep(2)
    print("second in func_a!")
    await asyncio.sleep(2)
    print("third in func_a!")


async def func_b():
    print("first in func_b!")
    await asyncio.sleep(2)
    print("second in func_b!")
    await asyncio.sleep(2)
    print("third in func_b!")


tasks = [asyncio.ensure_future(func_a()),
         asyncio.ensure_future(func_b())]
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.wait(tasks))

first in func_a!
first in func_b!
second in func_a!
second in func_b!
third in func_a!
third in func_b!

1.2.5 协程的意义

在一个线程中，如果遇到IO等待时间，利用该空闲时间去处理其余任务。

2. 异步编程（asyncio模块）

2.1 事件循环

2.1.1 定义

死循环：检测并执行某些代码。

import asyncio

# 获取一个事件循环
loop = asyncio.get_event_loop()
#将任务放到任务列表
loop.run_until_complete(tasks)

2.2 协程函数（async关键字） & 携程对象

协程函数
async def 函数名
携程对象
	执行携程函数得到的对象 func()
async func():
	pass
result = func() # 得到协程对象时，函数内部代码不会执行。

执行协程对象

import asyncio
async def func():
	pass
asyncio.run(func())

2.3 await关键字

await 后接可等待的对象
	 （协程对象、Future对象、Task对象）
	  即IO等待

# 举例1
import asyncio
async def func():
	print("开始执行！")
	response = await asyncio.sleep(2)
	print("已结束！",response)
asyncio.run(func())

# 举例2
import asyncio

async def fun_a():
    print("func_a:1!")
    await asyncio.sleep(2)
    print("func_a:2!")
    return "func_a:over!"

async def fun_b():
    print("func_b:1")
    response = await fun_a()
    print(response)

asyncio.run(fun_b())

结果为：
func_b:1
func_a:1!
func_a:2!
func_a:over!

2.4 Task对象

在事件循环中添加多个任务。

Task用于并发调度协程，
asyncio.create_task(协程对象)

import asyncio

async def fun_a():
    print("func_a:1!")
    await asyncio.sleep(2)
    print("func_a:2!")
    return "func_a:over!"

async def fun_b():
    print("func_b:1")
    tasks = [asyncio.create_task(fun_a()),
             asyncio.create_task(fun_a()),
             ]
    done,pending = await asyncio.wait(tasks)
    print(done)

asyncio.run(fun_b())

func_b:1
func_a:1!
func_a:1!
func_a:2!
func_a:2!

2.5 future 对象

Task继承Future对象，Task的await结果由Future处理而来。（很少使用）

2.6 concurrent.futures.Future对象

使用进程池或线程池实现异步操作。

from concurrent.futures import Future
from concurrent.futures.thread import ThreadPoolExecutor
from concurrent.futures.process import ProcessPoolExecutor


def func(val):
    print(val)
    return val

thread_pool = ThreadPoolExecutor(max_workers=5)

for i in range(10):
    fut = thread_pool.submit(func,i)
    print(fut)

协程与线程（进程）的交叉使用

import asyncio
from concurrent.futures import Future
from concurrent.futures.thread import ThreadPoolExecutor
from concurrent.futures.process import ProcessPoolExecutor


async def func_a():
    print("func_a:1!")
    await asyncio.sleep(2)
    print("func_a:2!")
    return 200

def func_b(val):
    print(val)
    print("I am in func_b!")
    return 100

async def main():
    print("main func:1!")
    result = await func_a()
    print(result)

    loop = asyncio.get_running_loop()
    # 1、使用线程调用func_b
    future = loop.run_in_executor(None,func_b,2011)

    # # 2、使用线程调用func_b
    # with ThreadPoolExecutor(max_workers=5) as executor:
    #     future = loop.run_in_executor(executor, func_b, 2011)
    #
    # # 3、使用进程调用func_b
    # with ProcessPoolExecutor(max_workers=5) as executor:
    #     future = loop.run_in_executor(executor, func_b, 2011)

    response = await future

    print(response)



asyncio.run(main())

main func:1!
func_a:1!
func_a:2!
200
2011
I am in func_b!
100

案例（asyncio+不支持异步的模块）

// An highlighted block
import asyncio
from concurrent.futures.thread import ThreadPoolExecutor
import requests
import random


async def spider(url):
    loop = asyncio.get_running_loop()
    print("start to download {}".format(url))

    with ThreadPoolExecutor(max_workers=5) as executor:
        result = loop.run_in_executor(executor,requests.get,url)
    response = await result

    print("end to download {}".format(url))

    with open("{}.png".format(random.choice([i for i in range(1000)])),"wb") as f:
        f.write(response.content)


urls = ["https://image.baidu.com/search/detail?ct=503316480&z=0&ipn=false&word=%E4%BA%94%E6%9C%88%E5%A4%A9%E5%9B%BE%E7%89%87&step_word=&hs=0&pn=0&spn=0&di=101860&pi=0&rn=1&tn=baiduimagedetail&is=0%2C0&istype=0&ie=utf-8&oe=utf-8&in=&cl=2&lm=-1&st=undefined&cs=548937810%2C3900574322&os=2218065150%2C2242708278&simid=4253052539%2C539865446&adpicid=0&lpn=0&ln=1334&fr=&fmq=1642142581695_R&fm=&ic=undefined&s=undefined&hd=undefined&latest=undefined©right=undefined&se=&sme=&tab=0&width=undefined&height=undefined&face=undefined&ist=&jit=&cg=star&bdtype=0&oriquery=&objurl=https%3A%2F%2Fgimg2.baidu.com%2Fimage_search%2Fsrc%3Dhttp%3A%2F%2Fpic1.zhimg.com%2F50%2Fv2-3b13501c1397e36b11a0efb9fc2ca195_hd.jpg%26refer%3Dhttp%3A%2F%2Fpic1.zhimg.com%26app%3D2002%26size%3Df9999%2C10000%26q%3Da80%26n%3D0%26g%3D0n%26fmt%3Djpeg%3Fsec%3D1644734586%26t%3Dedf6d65698f1c31375d2689bfcd59533&fromurl=ippr_z2C%24qAzdH3FAzdH3Fooo_z%26e3Bziti7_z%26e3Bv54AzdH3Fq7jfpt5gAzdH3Fdbd00mm9aAzdH3Fwgfoj6AzdH3F00d9nmdmc&gsm=1&rpstart=0&rpnum=0&islist=&querylist=&nojc=undefined&dyTabStr=MCwzLDEsNiw0LDIsNSw3LDgsOQ%3D%3D",
        "https://image.baidu.com/search/detail?ct=503316480&z=0&ipn=false&word=%E4%BA%94%E6%9C%88%E5%A4%A9%E5%9B%BE%E7%89%87&step_word=&hs=0&pn=1&spn=0&di=70180&pi=0&rn=1&tn=baiduimagedetail&is=0%2C0&istype=0&ie=utf-8&oe=utf-8&in=&cl=2&lm=-1&st=undefined&cs=301737693%2C3447630243&os=1460876122%2C3473358269&simid=4250352230%2C715662432&adpicid=0&lpn=0&ln=1334&fr=&fmq=1642142581695_R&fm=&ic=undefined&s=undefined&hd=undefined&latest=undefined©right=undefined&se=&sme=&tab=0&width=undefined&height=undefined&face=undefined&ist=&jit=&cg=star&bdtype=0&oriquery=&objurl=https%3A%2F%2Fgimg2.baidu.com%2Fimage_search%2Fsrc%3Dhttp%3A%2F%2Fb-ssl.duitang.com%2Fuploads%2Fitem%2F201704%2F03%2F20170403095157_yfQ5i.thumb.700_0.jpeg%26refer%3Dhttp%3A%2F%2Fb-ssl.duitang.com%26app%3D2002%26size%3Df9999%2C10000%26q%3Da80%26n%3D0%26g%3D0n%26fmt%3Djpeg%3Fsec%3D1644734586%26t%3Db079866dcc8b21bbdae0450fea819256&fromurl=ippr_z2C%24qAzdH3FAzdH3Fooo-kjpw8_z%26e3B17tpwg2_z%26e3Bv54AzdH3Fks52AzdH3F%3Ft1%3D0n98n9bn9&gsm=1&rpstart=0&rpnum=0&islist=&querylist=&nojc=undefined&dyTabStr=MCwzLDEsNiw0LDIsNSw3LDgsOQ%3D%3D",
        "https://image.baidu.com/search/detail?ct=503316480&z=0&ipn=false&word=%E4%BA%94%E6%9C%88%E5%A4%A9%E5%9B%BE%E7%89%87&step_word=&hs=0&pn=4&spn=0&di=164890&pi=0&rn=1&tn=baiduimagedetail&is=0%2C0&istype=0&ie=utf-8&oe=utf-8&in=&cl=2&lm=-1&st=undefined&cs=554303154%2C2476822283&os=2894650807%2C2922528529&simid=3343407959%2C137235789&adpicid=0&lpn=0&ln=1334&fr=&fmq=1642142581695_R&fm=&ic=undefined&s=undefined&hd=undefined&latest=undefined©right=undefined&se=&sme=&tab=0&width=undefined&height=undefined&face=undefined&ist=&jit=&cg=star&bdtype=0&oriquery=&objurl=https%3A%2F%2Fgimg2.baidu.com%2Fimage_search%2Fsrc%3Dhttp%3A%2F%2Finews.gtimg.com%2Fnewsapp_match%2F0%2F11861039407%2F0.jpg%26refer%3Dhttp%3A%2F%2Finews.gtimg.com%26app%3D2002%26size%3Df9999%2C10000%26q%3Da80%26n%3D0%26g%3D0n%26fmt%3Djpeg%3Fsec%3D1644734586%26t%3D7dcf718e3ef7b029dbfbb0312ac5c301&fromurl=ippr_z2C%24qAzdH3FAzdH3Fh7wtkw5_z%26e3Bqq_z%26e3Bv54AzdH3FfAzdH3FdadaamadAaHbUDaa%3F6juj6%3Dfrt1j6&gsm=1&rpstart=0&rpnum=0&islist=&querylist=&nojc=undefined&dyTabStr=MCwzLDEsNiw0LDIsNSw3LDgsOQ%3D%3D",
        "https://image.baidu.com/search/detail?ct=503316480&z=0&ipn=false&word=%E4%BA%94%E6%9C%88%E5%A4%A9%E5%9B%BE%E7%89%87&step_word=&hs=0&pn=18&spn=0&di=125620&pi=0&rn=1&tn=baiduimagedetail&is=0%2C0&istype=0&ie=utf-8&oe=utf-8&in=&cl=2&lm=-1&st=undefined&cs=3919725283%2C4047357414&os=1077457953%2C3020600078&simid=4210168092%2C663966287&adpicid=0&lpn=0&ln=1334&fr=&fmq=1642142581695_R&fm=&ic=undefined&s=undefined&hd=undefined&latest=undefined©right=undefined&se=&sme=&tab=0&width=undefined&height=undefined&face=undefined&ist=&jit=&cg=star&bdtype=0&oriquery=&objurl=https%3A%2F%2Fgimg2.baidu.com%2Fimage_search%2Fsrc%3Dhttp%3A%2F%2Fb-ssl.duitang.com%2Fuploads%2Fitem%2F201707%2F21%2F20170721002211_RkCmF.thumb.700_0.jpeg%26refer%3Dhttp%3A%2F%2Fb-ssl.duitang.com%26app%3D2002%26size%3Df9999%2C10000%26q%3Da80%26n%3D0%26g%3D0n%26fmt%3Djpeg%3Fsec%3D1644734693%26t%3D56b4d33f55e7483f69343b4e9ebaab21&fromurl=ippr_z2C%24qAzdH3FAzdH3Fooo_z%26e3B17tpwg2_z%26e3Bv54AzdH3Fks52AzdH3F%3Ft1%3Dbalcllal0&gsm=13&rpstart=0&rpnum=0&islist=&querylist=&nojc=undefined&dyTabStr=MCwzLDEsNiw0LDIsNSw3LDgsOQ%3D%3D",
        "https://image.baidu.com/search/detail?ct=503316480&z=0&ipn=false&word=%E4%BA%94%E6%9C%88%E5%A4%A9%E5%9B%BE%E7%89%87&step_word=&hs=0&pn=38&spn=0&di=69740&pi=0&rn=1&tn=baiduimagedetail&is=0%2C0&istype=0&ie=utf-8&oe=utf-8&in=&cl=2&lm=-1&st=undefined&cs=3698734864%2C2135288157&os=1009166602%2C1961084779&simid=3421568423%2C430913388&adpicid=0&lpn=0&ln=1334&fr=&fmq=1642142581695_R&fm=&ic=undefined&s=undefined&hd=undefined&latest=undefined©right=undefined&se=&sme=&tab=0&width=undefined&height=undefined&face=undefined&ist=&jit=&cg=star&bdtype=0&oriquery=&objurl=https%3A%2F%2Fgimg2.baidu.com%2Fimage_search%2Fsrc%3Dhttp%3A%2F%2F5b0988e595225.cdn.sohucs.com%2Fimages%2F20181103%2Fac15613329de46099b3bc0d5398ff9ea.jpeg%26refer%3Dhttp%3A%2F%2F5b0988e595225.cdn.sohucs.com%26app%3D2002%26size%3Df9999%2C10000%26q%3Da80%26n%3D0%26g%3D0n%26fmt%3Djpeg%3Fsec%3D1644734709%26t%3D60e503e73c3c9a23d100ebd435e17c48&fromurl=ippr_z2C%24qAzdH3FAzdH3Fooo_z%26e3Bf5i7_z%26e3Bv54AzdH3FwAzdH3Fd0dlcmamd_m09baa&gsm=26&rpstart=0&rpnum=0&islist=&querylist=&nojc=undefined&dyTabStr=MCwzLDEsNiw0LDIsNSw3LDgsOQ%3D%3D",]

task = [spider(url) for url in urls]

asyncio.run(asyncio.wait(task))

2.7 异步迭代器

不常用

2.8 异步上下文管理器

import asyncio
async def main():
	async with AsyncManager() as f:
		f.do_sth()
asyncio.run(main())

3. uvloop

使用uvloop替代asyncio的默认事件循环，效率高出两倍
pip install uvloop


import asyncio
import uvloop
asyncion.set_event_loop_policy(uvloop.EventLoopPolicy())

4. 案例

4.1 连接MySQL

pip insatll aiomysql

4.2 FastAPI框架

pip insatll fastapi
pip insatll uvicorn

by CyrusMay 2022 01 14

世界再大
不过你和我
用最小回忆
堆成宇宙
—————五月天（因为你 所以我）—————