第六章 Scrapy框架(二十七) 2020-03-29

二十七、 分布式爬虫– 实战– 链家网全国房源信息爬取(3


pipeline.py 代码


import json

 

 

class LjPipeline(object):

    def __init__(self):

        self.fp = open("houses.txt",'w', encoding='utf-8')

 

    def process_item(self, item, spider):

        self.fp.write(json.dumps(dict(item),ensure_ascii=False)+"\n")

        return item

 

    def close_spider(self, spider):

        self.fp.close()


settings.py 设置

 

ITEM_PIPELINES= {

   'lj.pipelines.LjPipeline': 300,

}


创建 start.py 代码


from scrapy import cmdline

 

cmdline.execute("scrapycrawl house".split(" "))


续上例,house.py 示例代码


import scrapy

from ..items import LjItem

import re

import json

 

 

class HouseSpider(scrapy.Spider):

    name = 'house'

    allowed_domains = ['lianjia.com']

    start_urls =['https://www.lianjia.com/city/']

 

    def parse(self, response):

        city_tags =response.css(".city_list_ul a")

        for city_tag in city_tags:

            city_url = city_tag.css("::attr(href)").get()

            city_name =city_tag.css("::text").get()

            item = LjItem(city=city_name)

            yield scrapy.Request(city_url+"ershoufang/",callback=self.parse_region_list, meta={"item": item})

 

    def parse_region_list(self, response):

       # 解析行政区的url

        item = response.meta.get('item')

        region_tags =response.css("div[data-role='ershoufang'] .a")

        for region_tag in region_tags:

            region_url =region_tag.css("::attr(href)").get()

            region_name =region_tag.css("::text").get()

            item['region'] = region_name

            yield scrapy.Request(response.urljoin(region_url), callback=self.parse_house_page,meta={"item": item})

 

    def parse_house_page(self, response):

       # 翻页

        page_data =response.css("div[comp-module='page']::attr(page-data)").get()

        totalPage =json.loads(page_data)['totalPage']

        for x in range(1, totalPage + 1):

            yield scrapy.Request(response.url +"pg" + str(x), callback=self.parse_house_list,meta={"item":response.meta.get('item')})

 

    def parse_house_list(self, response):

       # 解析房源列表

        item =response.meta.get("item")

        detail_urls = response.css(".sellListContentli>a::attr(href)").getall()

        for detail_url in detail_urls:

            result =re.search(r'/ershoufang/\d+\.html', detail_url)

            if result:

                yieldscrapy.Request(detail_url, callback=self.parse_house, meta={"item":item})

 

    def parse_house(self, response):

       # 解析房源详情页

        item = response.meta.get('item')

        item['title'] =response.css("h1.main::text").get()

        item['total_price'] =response.css(".price.total::text").get()

        item['unit_price'] =response.css(".unitPriceValue::text").get()

        item['house_type'] =response.css(".content ul li:nth-child(1)::text").get()

        item['orientation'] =response.css(".content ul li:nth-child(7)::text").get()

        item['full_area'] =response.css(".content ul li:nth-child(3)::text").get()

        item['inside_area'] =response.css(".content ul li:nth-child(5)::text").get()

        item['years'] =response.css(".area .subInfo::text").get()

        yield item



上一篇文章 第六章 Scrapy框架(二十六) 2020-03-28 地址:

https://www.jianshu.com/p/cd9593e50b3a

下一篇文章 第六章 Scrapy框架(二十八) 2020-03-30 地址:

https://www.jianshu.com/p/fe9e9f6505c6



以上资料内容来源网络,仅供学习交流,侵删请私信我,谢谢。

你可能感兴趣的:(第六章 Scrapy框架(二十七) 2020-03-29)