pycharm运行scrapy框架爬取豆瓣电影250可能遇到的问题

一、首先cmd命令创建scrapy项目：scrapy startproject 项目名

        --然后cmd命令创建scrapy爬虫任务：scrapy genspider 爬虫任务名 域名.com

        如果需要在pycharm中运行scrapy框架,就在scrapy.cfg文件的同级目录下创建一个可执行文件  ：文件名（随意起)

二、打开settings文件，设置用户代理：这个是初始的代码：

1.UA设置：#USER_AGENT = 'douban2 (+http://www.yourdomain.com)'，需要把注释关掉，uA改为自己电脑的用户代理值，例如：

USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.8 Safari/537.36'

2.关闭爬虫协议：初始是这样的：#ROBOTSTXT_OBEY = True,改为 ROBOTSTXT_OBEY = False

3.设置下载延迟，防止请求过快导致IP被封：DOWNLOAD_DELAY = 0.5

4.设置Cookie:COOKIES_ENABLED = False

 DEFAULT_REQUEST_HEADERS = { # 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', # 'Accept-Language': 'en', 'Cookie':'cookie的值'

'Referer':'跳转参数' }

5.打开管道：

ITEM_PIPELINES = { 'douban2.pipelines.Douban2Pipeline': 300, } #ITEM_PIPELINES要解开注释才能把数据存储到文件

下面是scrapy爬取豆瓣电影250案例

任务文件movie.py

import scrapy
from ..items import Douban2Item  # 从items.py中导入Douban2Item类来装载数据


class Movie2Spider(scrapy.Spider):
    name = 'movie2'
    allowed_domains = ['douban.com']
    start_urls = ['https://movie.douban.com/top250']

    def parse(self, response):
        # 提取数据
        # 另外一种方法：title_list = response.xpath('//span[@class="title"][1]/text()').extract()
        #             score_list = response.xpath('//span[@class="rating_num"]/text()').extract()
        title_list = response.xpath('//span[@class="title"][1]/text()').getall()  # 电影名称
        print(title_list)
        score_list = response.xpath('//span[@class="rating_num"]/text()').getall()  # 电影评分
        print(score_list)
        url_list = response.xpath('//div[@class="hd"]/a/@href').getall()  # 详情页url
        # 构建数据载体,装载数据方式类似字典
        
        for i in range(len(url_list)):
            item_ = Douban2Item()
            item_['title_'] = title_list[i]
            item_['score_'] = score_list[i]
            # 详情页的url需要手动构造requst对象来发送请求，用yield把requstq对象传给引擎
            #               url          解析方法       跨方法传递数据的方法 装载数据的对象作为meta字典的值
            yield scrapy.Request(url_list[i], callback=self.parse_url, meta={'fish': item_})

    # 手动构造详情页的解析方法
    def parse_url(self, response):
        # xpath语法不通用 需要两种语句才能提取 这里提取到的text是单个的简介，而且分成好几个部分的，所以需要拼接
        text_list = response.xpath('//div[@class="indent"]/span[2]/text()|//div[@class="indent"]/span[1]/text()').getall()
        text_ = ''.join(text_list)  # yield单个传递请求，所以解析出来的是单个电影的简介
        text_ = text_.replace('\u3000','')
        text_ = text_.replace('\n','')
        text_ = text_.replace(' ','')
        print(text_)
        # 另外一种接收item对象数据的方法： item_ = response.meta['fish']
        item_ = response.meta.get('fish')
        # 将text_数据装进运输载体
        item_['text_'] = text_
        # 把数据交给引擎，引擎再交给管道
        yield item_

pipelines.py

import json


class Douban2Pipeline:
    def __init__(self):
        self.file = open('douban250_2.json', 'w', encoding='UTF-8')
        print('文件douban250_2.json >>>>打开了')

    def process_item(self, item, spider):
        # 数据载体对象字典转换
        dict_ = dict(item)
        # 字典数据转为json数据格式，存入文件
        json_data = json.dumps(dict_, ensure_ascii=False) + ',\n'
        self.file.write(json_data)
        print('一条数据被写入文件douban250_2.json')
        return item  # 多管道时，用于把数据载体对象传递给下一个管道

    def close_spider(self, spider):
        self.file.close()
        print('文件douban250_2.json >>>>关闭了')

setting.py

# Scrapy settings for douban2 project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     https://docs.scrapy.org/en/latest/topics/settings.html
#     https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
#     https://docs.scrapy.org/en/latest/topics/spider-middleware.html

BOT_NAME = 'douban2'

SPIDER_MODULES = ['douban2.spiders']
NEWSPIDER_MODULE = 'douban2.spiders'


# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'douban2 (+http://www.yourdomain.com)'
USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.8 Safari/537.36'
# Obey robots.txt rules
ROBOTSTXT_OBEY = False  # 爬虫协议关闭一下

# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32

# Configure a delay for requests for the same website (default: 0)
# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
DOWNLOAD_DELAY = 0.5
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default)
# COOKIES_ENABLED = False
COOKIES_ENABLED = False

# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False

# Override the default request headers:
DEFAULT_REQUEST_HEADERS = {
#   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
#   'Accept-Language': 'en',
    'Cookie':'ll="118286"; bid=47FMkTWzZI8; push_noty_num=0; push_doumail_num=0; __utmv=30149280.21661; _vwo_uuid_v2=D2BD448DF51E5648418E0B28487BF4D1B|b06c2cda3cc6f9239f9ba52c2539bef8; __gads=ID=31827dbd1130432c-22caf2faacca0016:T=1628049782:RT=1628049782:S=ALNI_Mb469ig1mvKwkQiQ6H9iVtUkKueeg; __yadk_uid=9Q8Ffj5vYmiFTyr8HsXK5mC03cMANKSo; __utmz=30149280.1631278427.25.8.utmcsr=baidu|utmccn=(organic)|utmcmd=organic; __utmz=223695111.1631278427.25.9.utmcsr=baidu|utmccn=(organic)|utmcmd=organic; dbcl2="216614413:uIcIZKCE9f8"; ck=nmba; _pk_ref.100001.4cf6=%5B%22%22%2C%22%22%2C1631338051%2C%22https%3A%2F%2Fwww.baidu.com%2Flink%3Furl%3Di6tGjC8wPEiSsu-JxdkiWUBcEQU81foPjJE0zHiZOUG7-rZEPYg7qK1XSHyEKgvH%26wd%3D%26eqid%3Da11ed622000282fc00000006613b5539%22%5D; _pk_id.100001.4cf6=51a2307cd23f3f40.1627965241.26.1631338051.1631280367.; _pk_ses.100001.4cf6=*; __utma=30149280.2139467737.1627965184.1631278427.1631338051.26; __utmb=30149280.0.10.1631338051; __utmc=30149280; __utma=223695111.889268973.1627965241.1631278427.1631338051.26; __utmb=223695111.0.10.1631338051; __utmc=223695111',
    'Referer':'https://movie.douban.com/top250'
}

# Enable or disable spider middlewares
# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
#    'douban2.middlewares.Douban2SpiderMiddleware': 543,
#}

# Enable or disable downloader middlewares
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
#DOWNLOADER_MIDDLEWARES = {
#    'douban2.middlewares.Douban2DownloaderMiddleware': 543,
#}

# Enable or disable extensions
# See https://docs.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
#    'scrapy.extensions.telnet.TelnetConsole': None,
#}

# Configure item pipelines
# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
   'douban2.pipelines.Douban2Pipeline': 300,
}   #ITEM_PIPELINES要解开注释才能把数据存储到文件

# Enable and configure the AutoThrottle extension (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False

# Enable and configure HTTP caching (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
# FEED_EXPORT_ENCODING = 'UTF-8'

注意：cookie值需要更新的。如果没什么改动，直接运行scrapy框架发现爬取不到数据，那重新复制一下cookie值就了，例如出现这样的：

 

 pipelines.py

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html


# useful for handling different item types with a single interface
from itemadapter import ItemAdapter
import json


class Douban2Pipeline:
    def __init__(self):
        self.file = open('douban250_2.json', 'w', encoding='UTF-8')
        print('文件douban250_2.json >>>>打开了')

    def process_item(self, item, spider):
        # 数据载体对象字典转换
        dict_ = dict(item)
        # 字典数据转为json数据格式，存入文件
        json_data = json.dumps(dict_, ensure_ascii=False) + ',\n'
        self.file.write(json_data)
        print('一条数据被写入文件douban250_2.json')
        return item  # 多管道时，用于把数据载体对象传递给下一个管道

    def close_spider(self, spider):
        self.file.close()
        print('文件douban250_2.json >>>>关闭了')

items.py

# Define here the models for your scraped items
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/items.html

import scrapy


class Douban2Item(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    title_ = scrapy.Field()
    score_ = scrapy.Field()
    text_ = scrapy.Field()
    pass

-----如果有帮到你，麻烦给个赞，哈哈。。。