python异步协程简单爬虫

其中 1.txt是保存的网页源代码 放在当前文件夹

import urllib.request
import re
import gevent
from gevent.pool import Pool
from gevent import monkey
monkey.patch_all()


def jpg_dy():
    # 提取网页图片链接
    # global pics
    with open("1.txt","r") as f:
        a = f.read()
    # print(a)
    pic = re.findall(r"https://[^:]*\.jpg", a)
    print(pic)
    return pic
def download(pic,count_a):
    # 将图片保存


    req = urllib.request.urlopen(pic)

    img_content = req.read()

    with open("斗鱼"+str(count_a)+".jpg", "wb") as f1:
        f1.write(img_content)




def main():
    """爬取斗鱼"""
    h = 1
    po = Pool(10)
    for i in jpg_dy():
        h+=1
        po.apply_async(download,(i,h))
    print("正在爬取图片")

    po.join()


if __name__ == '__main__':
    main()