pyspider web爬虫框架简单使用

一、pyspider简介

参考文档：

http://docs.pyspider.org/

二、需要安装的依赖包

sudo apt-get install python

sudo apt-get install python-dev

sudo apt-get install python-distribute

sudo apt-get install python-pip

sudo apt-get install libcurl4-openssl-dev

sudo apt-get install libxml2-dev

sudo apt-get install libxslt1-dev

sudo apt-get install python-lxml libssl-dev

sudo apt-get install zlib1g-dev

sudo apt-get install phantomjs

pip3 install pyspider

启动

pyspider all

代码样例

from pyspider.libs.base_handler import *
import pymongo,pymysql

class Handler(BaseHandler):
"""
Handler 就是 pyspider 爬虫的主类，我
们可以在此处定义爬取、解析、存储的逻辑。
整个爬虫的功能只需要一个 Handler 即可完成
"""

#crawl_config 属性。我们可以将本项目的
#所有爬取配置统一定义到这里，如定义
#Headers、设置代理等，配置之后全局生效
crawl_config = {
}
#mongodb数据库连接
mongocli = pymongo.MongoClient(host='localhost', port=27017)
db = mongocli['jobbole']
jobbolearticle = db['jobbolearticle']

#mysql数据库连接
client = pymysql.Connect(
'localhost','root','xxx',
'xxx',charset='utf8'
)
cursor = client.cursor()

#on_start() 方法是爬取入口，初始的爬取
#请求会在这里产生
@every(minutes=24 * 60)
def on_start(self):
self.crawl('http://blog.jobbole.com/all-posts/', callback=self.index_page)

@config(age=10 * 24 * 60 * 60)
def index_page(self, response):
print(response)
for each in response.doc('a[class="page-numbers"]').items():
self.crawl(each.attr.href, callback=self.detail_page)

@config(priority=2)
def detail_page(self, response):
self.index_page(response)
list = response.doc('#archive .post.floated-thumb')
# print(list)
for item in list.items():
print('拿到了数据')
print(type(item))
title = item('a.archive-title').text()
url = item('a.archive-title').attr.href
print(title,url)
return {
'title':title,
'url':url,
}

#方法中return的结果会执行on_result
def on_result(self,result):
#可以在这里做数据的持久化
print(result)

#mysql数据库存储
sql = """
INSERT INTO jobbole()
VALUE (%s,%s)
"""
try:
self.cursor.execute(sql,[result['title'],result['url']])
self.client.commit()
except Exception as err:
print(err)
self.client.rollback()

#mongodb数据库存储
self.jobbolearticle.insert(result)