Python爬虫 Scrapy框架基础 讲解及教程

流程

将数据存储到Mongodb数据库:

#将数据存储到 mongo数据库
import pymongo
from jobboleproject.items import JobboleprojectItem

#改造版,将配置信息写在配置文件里面
class JobboleprojectPipeline(object):

    def __init__(self,mongo_host,mongo_port,mongo_db):
        self.mongoConn = pymongo.MongoClient(mongo_host,mongo_port)
        #切换到指定的数据库中
        self.db = self.mongoConn[mongo_db]
        #获取到数据库下的集合   更改集合名字的地方
        self.col = self.db.student

    @classmethod
    def from_crawler(cls, crawler):
        return cls(
            mongo_host=crawler.settings.get('MONGODB_HOST'),
            mongo_port=crawler.settings.get('MONGODB_PORT'),
            mongo_db = crawler.settings.get('MONGODB_DB')
        )

    def open_spider(self,spider):
        # 爬虫开启的时候调用，只调用一次
        print('爬虫文件开始执行',spider.name)

    def process_item(self, item, spider):
        print('管道文件我来了')
        self.col.insert(dict(item))
        return item

    def close_spider(self,spider):
        # 爬虫结束的时候会调用，只调用一次
        print('爬虫文件结束执行',spider.name)

将数据存储到Mysql数据库:

# 将数据存储到mysql数据库
import  pymysql
from jobboleproject.items import JobboleprojectItem
class JobboleprojectPipeline(object):

    def __init__(self,host,port,db,user,pwd):
        self.mysqlConn = pymysql.Connect(host=host,user=user,password=pwd,database=db,port=port,charset='utf8')
        self.cursor = self.mysqlConn.cursor()

    @classmethod
    def from_crawler(cls, crawler):
        return cls(
            host = crawler.settings.get('MYSQL_HOST'),
            port = crawler.settings.get('MYSQL_PORT'),
            db = crawler.settings.get('MYSQL_DB'),
            user = crawler.settings.get('MYSQL_USER'),
            pwd = crawler.settings.get('MYSQL_PWD'),
        )

    def open_spider(self,spider):
        # 爬虫开启的时候调用，只调用一次
        print('爬虫文件开始执行',spider.name)

    def process_item(self, item, spider):
        print('管道文件我来了')
        sql = """
        INSERT INTO abc(coverImage,link)
        VALUES (%s,%s)
        """
        try:
            self.cursor.execute(sql,(item['coverImage'],item['link']))
            self.mysqlConn.commit()
        except Exception as err:
            print(err)
            self.mysqlConn.rollback()
        return item

    def close_spider(self,spider):
        # 爬虫结束的时候会调用，只调用一次
        print('爬虫文件结束执行',spider.name)
        #关闭游标和关闭连接
        self.cursor.close()
        self.mysqlConn.close()


class JobboleprojectPipeline1(object):
    def process_item(self, item, spider):
        print('管道文件我来了1')
        return item

将数据存储为Json文件:

import json

#将数据写入json文件
class JobboleprojectPipeline(object):

    def __init__(self):
        self.file = open('data.json','a+')

    def open_spider(self,spider):
        # 爬虫开启的时候调用，只调用一次
        print('爬虫文件开始执行',spider.name)

    def process_item(self, item, spider):
        print('管道文件我来了')
        # with open('data.json','a+') as file:
        #     data = json.dumps(dict(item),ensure_ascii=False)+'\n'
        #     file.write(data)
        data = json.dumps(dict(item),ensure_ascii=False)+'\n'
        self.file.write(data)

        return item

    def close_spider(self,spider):
        # 爬虫结束的时候会调用，只调用一次
        print('爬虫文件结束执行',spider.name)
        self.file.close()

在settings文件中设置链接数据库代码

#mongodb配置信息
MONGODB_HOST = 'localhost'
MONGODB_PORT = 27017
MONGODB_DB = 'scrapy1803'

#mysql的配置信息
MYSQL_HOST = 'localhost'
MYSQL_PORT = 3306
MYSQL_USER = 'root'
MYSQL_PWD = 'ljh1314'
MYSQL_DB = 'class1803'