python异步协程(aiohttp,asyncio)

python异步协程

环境:python3.7.0

协程

协程，英文叫做 Coroutine，又称微线程，纤程，协程是一种用户态的轻量级线程。

协程本质上是个单进程，协程相对于多进程来说，无需线程上下文切换的开销，无需原子操作锁定及同步的开销，编程模型也非常简单。

使用协程来实现异步操作，发出一个请求之后，需要等待一定的时间才能得到响应，这个等待过程中，程序可以干许多其他的事情，等到响应得到之后才切换回来继续处理，这样可以充分利用 CPU 和其他资源，这就是异步协程的优势。

通过异步协程可以加快爬虫爬取文件的速度
其中用到aiohttp库

sudo pip3 install aiohttp

Name: aiohttp
Version: 3.4.1
Summary: Async http client/server framework (asyncio)
Home-page: https://github.com/aio-libs/aiohttp
Author: Nikolay Kim
Author-email: [email protected]
License: Apache 2
Location: /Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages
Requires: yarl, attrs, multidict, chardet, async-timeout
Required-by: 

以下分别是用异步与不用异步爬取unsplash首页图片的对比

unsplash.py

import requests
import os
import re
from time import time

class Spider(object):

    def __init__(self,n=10):
        self.headers={
            'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'
        }
        path='./download/unsplash'
        if not os.path.exists(path):
            os.mkdir(path)
            self.path=path
        else:
            self.path=path
        self.n=n
        self.num=1

    def getImagesLinks(self,page:int):
        url='https://unsplash.com/napi/photos'
        params={
            'page': page,
            'per_page': '12',
            'order_by': 'latest'
        }
        links=[]
        try:
            r=requests.get(url=url,params=params,timeout=60)
            r.raise_for_status()
            r.encoding=r.apparent_encoding
            for data in r.json():
                links.append(data['urls']['full'])
        except Exception as e:
            print(e.args)
        finally:
            return links

    def save_img(self,url):
        text=""
        try:
            r=requests.get(url,headers=self.headers,timeout=60)
            r.raise_for_status()
            text=r.content
        except Exception as e:
            print(e.args)
        finally:
            return text

    def download_img(self,url:str):
        url_split=re.split(r'\/|\?',url)
        try:
            filename=url_split[3]+'.jpg'
            if os.path.exists(self.path+'/'+filename):
                print('下载失败，文件已存在')
            else:
                with open(self.path+'/'+filename,'wb') as f:
                    f.write(self.save_img(url))
                    f.close()
                print('成功下载第%d张图片'%(self.num))
                self.num+=1
        except Exception as e:
            print(e.args)
        finally:
            pass

    def run(self):
        try:
            for i in range(1,self.n+1):
                urls=self.getImagesLinks(i)
                for url in urls:
                    self.download_img(url)
        except Exception as e:
            print(e.args)
        finally:
            pass



def main():
    start=time()
    spider=Spider(n=1)
    spider.run()
    end=time()
    print(end-start,'s')

if __name__ == '__main__':
    main()

2.png

unsplash.py爬取里unsplash首页的12张图片,共耗时62.375s

async_unsplash.py

import requests
import re
import os
import asyncio
import aiohttp
from time import time



class Spider(object):

    def __init__(self,n=10):
        self.headers = {
            'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'
        }
        path = './download/async_unsplash'
        if not os.path.exists(path):
            os.mkdir(path)
            self.path = path
        else:
            self.path = path
        self.n = n
        self.num=1

    def getImagesLinks(self,page:int):
        url='https://unsplash.com/napi/photos'
        params={
            'page': page,
            'per_page': '12',
            'order_by': 'latest'
        }
        links=[]
        try:
            r=requests.get(url=url,params=params,timeout=60)
            r.raise_for_status()
            r.encoding=r.apparent_encoding
            for data in r.json():
                links.append(data['urls']['full'])
        except Exception as e:
            print(e.args)
        finally:
            return links

    async def save_img(self,url):
        content=""
        try:
            async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(verify_ssl=False)) as session:
                response=await session.get(url,headers=self.headers,timeout=60)
                content=await response.read()
                await session.close()
        except Exception as e:
            print(e.args)
        finally:
            return content

    async def download_img(self,url:str):
        url_split = re.split(r'\/|\?', url)
        try:
            filename = url_split[3] + '.jpg'
            if os.path.exists(self.path + '/' + filename):
                print('下载失败，文件已存在')
            else:
                content=await self.save_img(url)
                with open(self.path + '/' + filename, 'wb') as f:
                    f.write(content)
                    f.close()
                print('成功下载第%d张图片' % (self.num))
                self.num+=1
        except Exception as e:
            print(e.args)
        finally:
            pass

    def run(self):
        try:
            for i in range(1, self.n + 1):
                urls = self.getImagesLinks(i)
                tasks=[asyncio.ensure_future(self.download_img(url)) for url in urls]
                loop=asyncio.get_event_loop()
                loop.run_until_complete(asyncio.wait(tasks))
        except Exception as e:
            print(e.args)
        finally:
            pass


def main():
    start=time()
    spider=Spider(n=1)
    spider.run()
    end=time()
    print(end-start,'s')

if __name__ == '__main__':
    main()

1.png

async_unsplash.py爬取里unsplash首页的12张图片,共耗时16.740s

通过上述两个事例，可以很明显看出异步协程对爬虫爬取文件速度的提升,对别的程序运行也有同样的作用

运行异步协程程序的时候,内存的占有比不用时大

GitHub地址