python 批量下载图片（协程）

要下载的图片网站

 1、总共多少页，得到每页的 url 列表

 

2、每页的图片详情的 ulr 列表（因为该高清大图在图片详情页，因此需要去图片详情页拿图片的url)

​​​​​​​

 

 

3、进入图片详情页，获取到图片url 然后下载。

 

完整代码如下：

import aiofiles
import aiohttp
import asyncio
import requests
from lxml import etree

# 下载单个图片
async def download_one(url):
    print("开始下载", url)
    name = url[0].split("/")[-1][:-4]
    head = {
        'Host': 'file.jiutuvip.com',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/114.0',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
        'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2',
        'Accept-Encoding': 'gzip, deflate, br',
        'Connection': 'keep-alive',
        'Upgrade-Insecure-Requests': '1',
        'Sec-Fetch-Dest': 'document',
        'Sec-Fetch-Mode': 'navigate',
        'Sec-Fetch-Site': 'none',
        'Sec-Fetch-User': '?1',
        'TE': 'trailers'
    }
    # 发送网络请求
    async with aiohttp.ClientSession() as session:
        async with session.get(url=url[0], headers=head) as resp:  # 相当于 requests.get(url=url[0], headers=head)
            # await resp.text() => resp.text
            content = await resp.content.read()  # => resp.content
            # 写入文件
            async with aiofiles.open('./img/' + name + '.webp', "wb") as f:
                await f.write(content)
    print("下载完毕")

# 获取图片的url
async def download(href_list):
    for href in href_list:
        async with aiohttp.ClientSession() as session:
            async with session.get(url=href) as child_res:
                html = await child_res.text()
                child_tree = etree.HTML(html)
                src = child_tree.xpath("//div[@class='img_box']/a/img/@src") # 选手图片地址 url 列表
                await download_one(src)

# 获取图片详情url
async def get_img_url(html_url):
    async with aiohttp.ClientSession() as session:
        async with session.get(url=html_url) as resp:
            html = await resp.text()
            tree = etree.HTML(html)
            href_list = tree.xpath("//div[@class='list-box-p']/ul/li/a/@href")  # 选手详情页 url 列表
            print(href_list)
            await download(href_list)
# 页面总页数
def get_html_url(url):
    page = 2
    response = requests.get(url=url)
    response.encoding = "utf-8"
    tree = etree.HTML(response.text)
    total_page = tree.xpath("//*[@id='pages']/a[12]/text()")  # 页面总页数
    print(total_page)
    html_url_list = []
    while page <= 4:  # int(total_page[0])  页数太多，本例只取第 2、3、4 页
        next_url = f"https://www.yeitu.com/meinv/xinggan/{page}.html"
        html_url_list.append(next_url)
        page += 1
    return html_url_list

async def main():
    # 拿到每页url列表
    url = "https://www.yeitu.com/meinv/xinggan/"
    html_url_list = get_html_url(url=url)  # 588个页面的url列表
    tasks = []
    for html_url in html_url_list:
        t = asyncio.create_task(get_img_url(html_url))  # 创建任务
        tasks.append(t)
    await asyncio.wait(tasks)

if __name__ == '__main__':
    event_loop = asyncio.get_event_loop()
    event_loop.run_until_complete(main())

结果如图所示：