数据爬取——http://pic.netbian.com 美女图片采集

import requests
from lxml import etree


headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.108 Safari/537.36"
}
url = 'http://pic.netbian.com/4kmeinv/index_%d.html'
for page in range(1,11):
    if page == 1:
        new_url = 'http://pic.netbian.com/4kmeinv/'
    else:
        new_url = format(url%page)
    response = requests.get(new_url,headers=headers)
    # response.encoding = 'utf-8'
    page_text = response.text

    # 解析：图片名称和图片的地址
    tree = etree.HTML(page_text)
    li_list = tree.xpath('//div[@class=\"slist\"]/ul/li')
    for li in li_list:
        img_title = li.xpath('./a/img/@alt')[0]
        # 通用解决中文乱码的处理方式
        img_title = img_title.encode('iso-8859-1').decode('gbk')
        img_url = 'http://pic.netbian.com'+li.xpath('./a/@href')[0]
        
        # 获取详情页面
        detail_page_text = requests.get(img_url, headers).text
        tree = etree.HTML(detail_page_text)
        
        filepath = 'http://pic.netbian.com' + tree.xpath('//a[@id="img"]/img/@src')[0]
        with open(f"{img_title}.jpg", "wb") as fp:
            fp.write(filepath.content)
print("下载结束！")