【Python笔记】爬虫连续多页爬取数据

爬虫四部曲：

1、获取网页；

2、解析网页文本；

3、提取需要的信息；

4、存储信息。

import requests
from lxml import etree
import xlsxwriter
url = 'https://...'
header = {'user-agent': '...'}
flag = True
workbook = xlsxwriter.Workbook('taoche_spider.xlsx')
worksheet = workbook.add_worksheet()
headings = ['**','**','**']
# worksheet.write(0,0,'**')
# worksheet.write(0, 1, '**')
# worksheet.write(0, 2, '**')
worksheet.write_row('A1',headings)
k = 1
while flag:
    html = requests.get(url, headers=header).text
    list = etree.HTML(html)
    lis = list.xpath('//ul[@class="gongge_ul"]/li')
    for selector in lis:
        name = selector.xpath('//div[@class="gongge_main"]/a/span/text()')
        originalprice = selector.xpath('//i[@class="Total brand_col"]/text()')
        unit = selector.xpath('//i[@class="Total brand_col"]/em/text()')
        for i in range(len(name)):
            worksheet.write(i + k, 0, name[i])
        for i in range(len(originalprice)):
            worksheet.write(i + k, 1, originalprice[i])
        for i in range(len(unit)):
            worksheet.write(i + k, 2, unit[i])
    try:
        i= len(unit)
        k = k + i
        next_url = list.xpath('//div/a[@class="pages-next"]/@href')[0]
        if next_url:
            url = next_url
    except:
        flag = False
worksheet.set_column("A:B",60)
worksheet.set_column("B:C",8)
workbook.close()

最终结果：