爬虫——多个url数据爬取

请求的手动发送

pipelines.py

class SecondPipeline(object):
    f = None

    def open_spider(self,spider):
        print('start')
        self.f = open('./qiubai.text','w',encoding='utf-8')

    def process_item(self, item, spider):
        self.f.write(item['author']+':'+ item['content'])
        return item

    def close_spider(self,spider):
        self.f.close()
        print('end')

qiubai.py

import scrapy
from second.items import SecondItem


class QiubaiSpider(scrapy.Spider):
    name = 'qiubai'
    # allowed_domains = ['www.qiushibaike.com']
    start_urls = ['https://www.qiushibaike.com/text/']

    url = 'https://www.qiushibaike.com/text/page/%d/'
    pageNum = 1

    def parse(self, response):
        print('正在爬虫')
        div_list = response.xpath("//div[@id='content-left']/div")
        for div in div_list:
            author = div.xpath('./div/a[2]/h2/text()').extract_first()
            content = div.xpath(".//div[@class='content']/span/text()").extract_first()

            items = SecondItem()
            items['author'] = author
            items['content'] = content

            yield items

        # 执行到此，第一个url已经获取成功，
        # 手动添加url,yield scrapy.Request，
        # callback 函数解析,可以自定义，也可以再次利用parse，不加（）,递归加条件
        # url = 'https://www.qiushibaike.com/text/page/2/'
        # yield scrapy.Request(url=url, callback=self.parse)

        if self.pageNum <= 13:
            self.pageNum += 1
            new_url = format(self.url % self.pageNum)
            yield scrapy.Request(url=new_url, callback=self.parse)

小结：

• 多个url手动添加，callback函数执行
• 页面布局一致时，循环调用解析函数递归注意终止条件
• yield scrapy.Request
• 注意面向对象 数据属性 与实例化对象 结合的特性！
• format函数的使用