scrapy项目各文件配置详细解析

无事做学了一下慕课网的scrapy爬虫框架，这里以豆瓣电影Top250爬虫为例子，课程用的MongoDB我这边使用的是mysql

1. settings文件参数含义

参数	含义
DOWNLOAD_DELAY = 0.5	下载延迟
DOWNLOADER_MIDDLEWARES= { # 这里的优先级不能相同 ‘crawler.middlewares.my_useragent’: 544,}	添加自己定义的伪装规则
ITEM_PIPELINES= { ‘crawler.pipelines.CrawlerPipeline’: 300,}	一定要开启pipeline

2. items设计数据结构

# -*- coding: utf-8 -*-
import scrapy


class CrawlerItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    serial_number = scrapy.Field()
    movie_name = scrapy.Field()
    introduce = scrapy.Field()
    star = scrapy.Field()
    evaluate = scrapy.Field()
    describe = scrapy.Field()

3. douban_spider编写爬虫xpath

# -*- coding: utf-8 -*-
# 豆瓣top50爬虫案例
import scrapy
from crawler.items import CrawlerItem

class DoubanSpiderSpider(scrapy.Spider):
    # 爬虫名，不能和项目名字重复
    name = 'douban_spider'
    # 允许的域名
    allowed_domains = ['movie.douban.com']
    # 入口url,扔到调度器里面
    start_urls = ['https://movie.douban.com/top250']

    # 默认解析方法
    def parse(self, response):
        # 循环电影条目
        movie_list = response.xpath("//div[@class='article']//ol[@class='grid_view']/li")
        for item in movie_list:
            # 导入items文件创建数据结构
            douban_item = CrawlerItem()
            # 写详细的xpath，进行数据解析
            douban_item['serial_number'] = int(item.xpath(".//div[@class='item']//em/text()").extract_first())
            douban_item['movie_name'] = item.xpath(".//div[@class='info']/div[@class='hd']/a/span[1]/text()").extract_first()
            content = item.xpath(".//div[@class='info']/div[@class='bd']/p[1]/text()").extract()
            # content_s = ''
            for i_content in content:
                # split（）以空格为分隔符，取最后一段
                content_s = ''.join(i_content.split())
                douban_item['introduce'] = content_s
            douban_item['star'] = float(item.xpath(".//span[@class='rating_num']/text()").extract_first())
            douban_item['evaluate'] = item.xpath(".//div[@class='star']/span[4]/text()").extract_first()
            douban_item['describe'] = item.xpath(".//span[@class='inq']/text()").extract_first()

            # 解析完，必须送到管道里
            yield douban_item
        # 解析下一页规则：检测是否有下一页，有连接就取，没有就不取
        next_link = response.xpath("//span[@class='next']/link/@href").extract()
        if next_link:
            nextlink = next_link[0]
            yield scrapy.Request("https://movie.douban.com/top250"+nextlink, callback=self.parse)

4. pipeline数据存储

# settings文件中设定数据库基本参数
# 数据库参数
sql_host = "localhost"
sql_port = 3306
sql_user = "root"
sql_passwd = "186186"
sql_db_name = 'scrapy_crawler'
# 数据库编码
sql_charset = 'utf8'
sql_use_unicode = True

# -*- coding: utf-8 -*-
import pymysql
from crawler.settings import sql_passwd,sql_db_name,sql_host,sql_user,sql_use_unicode,sql_charset,sql_port

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html


class CrawlerPipeline(object):
    def __init__(self):
        # 连接数据库
        self.connect = pymysql.connect(
            host=sql_host,
            port=sql_port,
            db=sql_db_name,
            user=sql_user,
            passwd=sql_passwd,
            charset=sql_charset,
            use_unicode=sql_use_unicode
        )
        # 拿到操作数据库的游标
        self.cursor = self.connect.cursor()


    def process_item(self, item, spider):
        # 执行插入操作
        self.cursor.execute(
            '''
            insert into douban_top250 VALUE (%s,%s,%s,%s,%s,%s)
            ''', (item['serial_number'], item['movie_name'], item['introduce'], item['star'], item['evaluate'], item['describe'])
        )
        self.connect.commit()
        return item

5. middlewares添加代理和agent池

# 编写代理ip，这里无法使用了过期了
class my_proxy(object):
    def process_request(self, request, spider):
        request.meta['proxy'] = 'http-cla.abuyun.com:9030'
        proxy_name_pass = b'H211EATS905745KC:F8FFBC929EB7D5A7'
        encode_pass_code = base64.b64encode(proxy_name_pass)
        request.headers['Proxy-Authrization'] = 'Basic '+ encode_pass_code.decode()

# 使用user-agent池使用不同的标识访问
class my_useragent(object):
    def process_request(self, request, spider):
        user_agent_list = [
            'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6',
            'Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5',
            'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER',
            'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)',
            'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11',
            'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
            'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)',
            'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11',
            'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; 360SE)',
            'Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)',
            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20',
            'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6',
            'Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10',
            'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER',
            'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1',
            'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)',
            'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.0.12) Gecko/20070731 Ubuntu/dapper-security Firefox/1.5.0.12',
            'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E; LBBROWSER)',
            'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1',
            'Mozilla/5.0 (iPhone; CPU iPhone OS 10_3 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.3 Mobile/14E277 Safari/603.1.30',
            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36'
        ]
        agent = random.choice(user_agent_list)
        request.headers['User_Agent'] = agent

6. 新建main.py作为项目执行入口

from scrapy import cmdline
cmdline.execute('scrapy crawl douban_spider'.split())

7. 数据库结构

8. 结果