python爬虫学习-scrapy爬取链家房源信息并存储（翻页）

爬取链家租房频道的房源信息，含翻页，含房间详情页的内容爬取。

items.py

import scrapy

class ScrapytestItem(scrapy.Item):
    # define the fields for your item here like:
    title = scrapy.Field()#房源名称
    price = scrapy.Field()#价格
    url = scrapy.Field()#详情页地址
    introduce_item = scrapy.Field()#房源描述

pipelines.py

import json

class ScrapytestPipeline(object):
    #打开文件
    def open_spider(self,spider):
        self.file = open('58_chuzu.txt','w',encoding='utf-8')
        print('文件被打开了')
    
    #写入文件
    def process_item(self, item, spider):
        line = '{}\n'.format(json.dumps(dict(item),ensure_ascii=False))
        self.file.write(line)
        return item
    
    #关闭文件
    def close_spider(self,spider):
        self.file.close()
        print('文件被关闭了')

spider

import scrapy
from ..items import ScrapytestItem
from scrapy.http import Request


class SpiderCity58Spider(scrapy.Spider):
    name = 'spider_city_58'#必不可少的爬虫名字
    allowed_domains = ['lianjia.com']
    start_urls = ['https://bj.lianjia.com/zufang/']

    def parse(self, response):
        #提取页面上的信息
        info_list = response.xpath('//*[@id="content"]/div[1]/div[1]/div')
        for i in info_list:
            item = ScrapytestItem()
            item['title'] = i.xpath('normalize-space(./div/p[1]/a/text())').extract()
            item['price'] = i.xpath('./div/span/em/text()').extract()
            url = i.xpath('./div/p[1]/a/@href').extract_first()#相对地址补全为绝对地址
            item['url'] = response.urljoin(url)
            
        #获取详情页的URL
            if item['url']:#判断URL是否为空
                yield Request(
                        item['url'],
                        callback = self.detail_parse,
                        meta = {'item':item},#只接受字典类型的赋值，将item传递给detali_parse()
                        priority = 10,
                        dont_filter = True
                        )
                
        #获取翻页URL
        for page in range(2,5):
            url = 'https://bj.lianjia.com/zufang/pg{}/'.format(str(page))#提取翻页链接
            test_request = Request(url,callback = self.parse)
            yield test_request

        #获取详情页的信息
    def detail_parse(self,response):
        item = response.meta['item']
        item['introduce_item'] = response.xpath('//*[@id="desc"]/ul/li/p[1]/text()').extract()
        return item

本来打算爬58的，但是58的反爬策略我还无法破解，所以换成了链家。

目前爬了4页，证明可行。后续添加时间控制等代码即可