scrapy爬取豆瓣电影 Top 250

1.网页

抓取电影名字、导演、时间、评分、评价人数、评论 

2.items

class DoubanItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    name = scrapy.Field()
    director = scrapy.Field()
    time = scrapy.Field()
    score = scrapy.Field()
    num = scrapy.Field()
    comment = scrapy.Field()

3.spider

import scrapy
from maitian.items import DoubanItem

class DoubanSpider(scrapy.Spider):
    name = 'douban'
    allowed_domains = ['movie.douban.com']
    start_urls = ['https://movie.douban.com/top250?start=0&filter=']

    def parse(self, response):
        item=DoubanItem()
        data_list=response.css('.info')
        for data in data_list:
            item['name']=data.css('.hd span::text').extract_first()
            item['director']=data.css('.bd p::text').extract()[0].strip()
            item['time']=data.css('.bd p::text').extract()[1].strip()
            item['score']=data.css('.bd .star .rating_num::text').extract_first()
            item['num']=data.xpath('.//div[@class="star"]/span[4]/text()').extract_first()
            item['comment']=data.css('.quote .inq::text').extract_first()
            yield item
        next_url=response.css('.next a::attr(href)').extract_first()
        if next_url is not None:
            yield scrapy.Request(response.urljoin(next_url))

4.存储数据

scrapy crawl douban -o douban.json -s FEED_EXPORT_ENCIDING=UTF-8
scrapy crawl douban -o douban.csv -s FEED_EXPORT_ENCIDING=UTF-8