Scrapy基于Mysql数据库异步写入数据库

1.在settings配置文件中设置连接参数

# 数据库地址
MYSQL_HOST = 'localhost'
# 数据库用户名:
MYSQL_USER = 'root'
#数据库密码
MYSQL_PASSWORD = '123456'
#数据库端口
MYSQL_PORT = 3306
#数据库名称
MYSQL_DBNAME = 'qisuu_data'
#数据库编码
MYSQL_CHARSET = 'utf8'

2.在pipelines文件中声明一个pipeline类

from pymysql import cursors
from twisted.enterprise import adbapi


class ToMysqlTwistedPipeline(object):

    # 初始化函数
    def __init__(self, db_pool):
        self.db_pool = db_pool

    # 从settings配置文件中读取参数
    @classmethod
    def from_settings(cls, settings):
        # 用一个db_params接收连接数据库的参数
        db_params = dict(
            host=settings['MYSQL_HOST'],
            user=settings['MYSQL_USER'],
            password=settings['MYSQL_PASSWORD'],
            port=settings['MYSQL_PORT'],
            database=settings['MYSQL_DBNAME'],
            charset=settings['MYSQL_CHARSET'],
            use_unicode=True,
            # 设置游标类型
            cursorclass=cursors.DictCursor
        )
        # 创建连接池
        db_pool = adbapi.ConnectionPool('pymysql', **db_params)

        # 返回一个pipeline对象
        return cls(db_pool)

    # 处理item函数
    def process_item(self, item, spider):
        # 把要执行的sql放入连接池
        query = self.db_pool.runInteraction(self.insert_into, item)
        # 如果sql执行发送错误,自动回调addErrBack()函数
        query.addErrback(self.handle_error, item, spider)
        
        # 返回Item
        return item
    
    # 处理sql函数    
    def insert_into(self,cursor,item):         
        # 创建sql语句        
        sql = "INSERT INTO qisuu (book_classsify,book_name,book_click,book_size,book_type,book_date,book_rate,book_author,book_run,book_onlin) VALUES ('{}','{}','{}','{}','{}','{}','{}','{}','{}','{}')".format(item['book_classsify'],item['book_name'],item['book_click'],item['book_size'],item['book_type'],item['book_date'],item['book_rate'],item['book_author'],item['book_run'],item['book_onlin'])        
        #执行sql语句        
        cursor.execute(sql)      
    #错误函数    
    def handle_error(self,failure,item,spider):        
        # #输出错误信息        
        print(failure)

3.启用pipeline

ITEM_PIPELINES = {
    'qisuuSpider.pipelines.ToMysqlTwistedPipeline': 10,
}