利用Scrapy抓取一带一路战略性支撑平台信息

本文首发于我的博客：http://gongyanli.com
代码传送门：https://github.com/Gladysgong/BeltandRoad
: https://www.jianshu.com/p/6fe4afa1b98a
CSDN: https://blog.csdn.net/u012052168/article/details/79761833

好久以前做的东西了，网站的数据也很容易拿到，最近想对自己的东西做一个总结，所以就有了这篇文章。 
我的目标是抓取一带一路战略支撑平台里面一些机构的数据，像联系电话、邮箱等等，简单看看网站的样子把。
首先获取列表项的所有url，其中包括一项翻页操作，拿到url后访问，获取里面的详细信息，就这么简单。

image

一、定义Items

`class BeltandRoadItem(scrapy.Item):
# collection = 'BeltandRoad'

name = scrapy.Field()  # 机构名称
intro = scrapy.Field()  # 机构简介
address = scrapy.Field()  # 机构地址
tel = scrapy.Field()  # 机构电话
fax = scrapy.Field()  # 机构传真
email = scrapy.Field()  # 机构邮箱
site = scrapy.Field()  # 机构网址`

二、爬虫模块

爬虫模块中包括翻页操作，并不复杂，就不详细说明。

---

`# -*- coding:utf-8 -*-
from scrapy.spiders import CrawlSpider, Rule
import scrapy
from ..items import BeltandRoadItem

# from scrapy.conf import settings

# 一带一路战略支撑平台
class BeltandRoadSpider(CrawlSpider):
    name = "BeltandRoadSpider"
    start_urls = ['http://ydyl.drcnet.com.cn/www/ydyl/channel.aspx?version=YDYL&uid=8011']

    # 解析url地址
    def parse(self, response):
        # urls = response.xpath('//div[@class="pub_right"]/ul/li/div[1]/a/@href').extract()
        urls = response.xpath('//ul[@id="ContentPlaceHolder1_WebPageDocumentsByUId1"]/li/div[1]/a/@href').extract()
        # institute_name = response.xpath('//ul[@class="left-nav"]/li/h3/a/text()').extract()

        for url in urls:
            # url = "http://ydyl.drcnet.com.cn/www/ydyl/" + url
            # print("1:", url)
            yield scrapy.Request(url=url, callback=self.parse_content)

        next_page = response.xpath(
            '//div[@id="ContentPlaceHolder1_WebPageDocumentsByUId1_PageRow"]/input[4]/@onclick').extract()
        next_page = str(next_page).split("\'")[1]
        print("next:", next_page)
        if next_page:
            next_page = "http://ydyl.drcnet.com.cn" + next_page
            yield scrapy.Request(url=next_page, callback=self.parse)

    # 解析内容
    def parse_content(self, response):
        item = BeltandRoadItem()
        name = response.xpath('//div[@id="disArea"]/strong/div/text()').extract()[0]  # 提取名称
        content = response.xpath('//div[@id="disArea"]/div[@id="docContent"]/p').xpath('string(.)').extract()  # 提取其他信息
        content = str(content).split('\'')
        item['name'] = name
        for i in range(len(content)):  # 过滤无效信息后，提取有用信息存储到item中
            if content[i] != '[' and content[i] != ']' and content[i] != ', ':
                print("xx:", content[i])
                if "简介" in content[i]:
                    item['intro'] = content[i]
                elif "地址" in content[i]:
                    item['address'] = content[i]
                elif "联系电话" in content[i]:
                    item['tel'] = content[i]
                elif "传真" in content[i]:
                    item['fax'] = content[i]
                elif "电子邮箱" in content[i]:
                    item['email'] = content[i]
                elif "网址" in content[i]:
                    item['site'] = content[i]
        yield item
`

三、构建pipelines

主要就是进行数据持久化操作，把数据存入MongoDB数据中。

---

`import pymongo
from scrapy.conf import settings
from .items import BeltandRoadItem

# 一带一路战略支撑平台
class BeltandRoadPipeline(object):
    def __init__(self, mongo_uri, mongo_db, mongo_port):
        self.mongo_uri = mongo_uri
        self.mongo_port = mongo_port
        self.mongo_db = mongo_db

    @classmethod
    def from_crawler(cls, crawler):
        return cls(mongo_uri=crawler.settings.get('MONGO_URI'),
                   mongo_port=crawler.settings.get('MONGO_PORT'),
                   mongo_db=crawler.settings.get('MONGO_DB')
                   )

    def open_spider(self, spider):
        self.client = pymongo.MongoClient(self.mongo_uri, self.mongo_port)
        self.db = self.client[self.mongo_db]
        self.BeltandRoad = self.db['BeltandRoad']

    def close_spider(self, spider):
        self.client.close()

    def process_item(self, item, spider):
        if isinstance(item, BeltandRoadItem):
            try:
                if item['name']:
                    item = dict(item)
                    # self.db[item.collection].insert(item)  运行这个代码时利用items中collection创建表，会提示插入失败，但是依然会插入到数据库？
                    self.BeltandRoad.insert(item)
                    print("插入成功")
                    return item
            except Exception as e:
                spider.logger.exception("插入失败")`

四、配置文件settings

# 激活pipelines
ITEM_PIPELINES = {
   'BeltandRoad.pipelines.BeltandRoadPipeline': 300,}

# 数据库配置
MONGO_URI = "127.0.0.1"  # 主机IP
MONGO_PORT = 27017  # 端口号
MONGO_DB = "Belt"  # 数据库名字