Python爬取伯乐在线网站

Python3.5+Scrapy爬取伯乐在线的博客文章

创建虚拟环境

mkvirtualenv -p C:\Users\Joseph\AppData\Local\Programs\Python\Python35\python.exe article_spider

pip install Twisted-17.5.0-cp35-cp35m-win_amd64.whl

pip install -i https://pypi.douban.com/simple/ scrapy

使用命令行创建scrapy项目

scrapy startproject ArticleSpider

创建基本模板

cd ArticleSpider
scrapy genspider jobbole blog.jobbole.com

启动scrapy

scrapy crawl jobbole

Windows下错误

错误：缺少win32模块
解决方法：pip install -i https://pypi.douban.com/simple pypiwin32

为了便于断点调试，新建main.py

from scrapy.cmdline import execute
import sys
import os

sys.path.append(os.path.dirname(os.path.abspath(__file__)))
execute(["scrapy", "crawl", "jobbole"])

注意：robots协议需要为False

在 settings.py中

# Obey robots.txt rules
ROBOTSTXT_OBEY = False

避免每次都获取页面，可以用shell调试scrapy

scrapy shell http://blog.jobbole.com/110287/

xpath提取元素

  title = response.xpath('//div[@class="entry-header"]/h1/text()').extract_first()

        create_date = response.xpath('//p[@class="entry-meta-hide-on-mobile"]/text()').extract_first().replace("·",
                                                                                                            "").strip()

        praise_nums = response.xpath("//span[contains(@class, 'vote-post-up')]/h10/text()").extract_first()

        fav_nums = response.xpath("//span[contains(@class, 'bookmark-btn')]/text()").extract_first()
        match_fav_nums = re.match(r".*?(\d+).*?", fav_nums)
        if match_fav_nums:
            fav_nums = match_fav_nums.group(1)

        comment_nums = response.xpath("//a[@href='#article-comment']/span/text()").extract_first()
        match_comment_nums = re.match(r".*?(\d+).*?", comment_nums)
        if match_comment_nums:
            comment_nums = match_comment_nums.group(1)

        content = response.xpath("//div[@class='entry']").extract_first()

        tag_list = response.xpath('//p[@class="entry-meta-hide-on-mobile"]/a/text()').extract()
        tag_list = [element for element in tag_list if not element.strip().endswith("评论")]
        tags = ",".join(tag_list)

css选择器提取元素

           title = response.css(".entry-header h1::text").extract_first()

        create_date = response.css("p.entry-meta-hide-on-mobile::text").extract_first().replace("·", "").strip()

        praise_nums = response.css(".vote-post-up h10::text").extract_first()

        fav_nums = response.css(".bookmark-btn::text").extract_first()
        match_fav_nums = re.match(r".*?(\d+).*?", fav_nums)
        if match_fav_nums:
            fav_nums = match_fav_nums.group(1)

        comment_nums = response.css("a[href='#article-comment'] span::text").extract_first()
        match_comment_nums = re.match(r".*?(\d+).*?", comment_nums)
        if match_comment_nums:
            comment_nums = match_comment_nums.group(1)

        content = response.css("div.entry").extract_first()

        tag_list = response.css("p.entry-meta-hide-on-mobile a::text").extract()
        tag_list = [element for element in tag_list if not element.strip().endswith("评论")]
        tags = ",".join(tag_list)

获取所有页面的需要数据

# -*- coding: utf-8 -*-
import re

import scrapy
from scrapy.http import Request
from urllib import parse

class JobboleSpider(scrapy.Spider):
    name = 'jobbole'
    allowed_domains = ['blog.jobbole.com']
    start_urls = ['http://blog.jobbole.com/all-posts/']

    def parse(self, response):
        """
        1.  获取文章列表页中的文章url并交给scrapy下载后进行解析
        2.  获取下一页的url并交给scrapy进行下载，下载完成后交给parse
        """

        # 解析列表页中的所有文章url并交给scrapy下载后进行解析
        post_urls = response.css("#archive .floated-thumb .post-thumb a::attr(href)").extract()
        for post_url in post_urls:
            yield Request(url=parse.urljoin(response.url, post_url), callback=self.parse_detail)

        # 提取下一页并交给scrapy下载
        next_url = response.css(".next.page-numbers::attr(href)").extract_first("")
        if next_url:
            yield Request(url=parse.urljoin(response.url, post_url), callback=self.parse)

    def parse_detail(self, response):
        # 提取文章具体字段
        title = response.xpath('//div[@class="entry-header"]/h1/text()').extract_first("")

        create_date = response.xpath('//p[@class="entry-meta-hide-on-mobile"]/text()').extract_first().replace("·",
                                                                                                               "").strip()

        praise_nums = response.xpath("//span[contains(@class, 'vote-post-up')]/h10/text()").extract_first("")

        fav_nums = response.xpath("//span[contains(@class, 'bookmark-btn')]/text()").extract_first("")
        match_fav_nums = re.match(r".*?(\d+).*?", fav_nums)
        if match_fav_nums:
            fav_nums = int(match_fav_nums.group(1))
        else:
            fav_nums = 0

        comment_nums = response.xpath("//a[@href='#article-comment']/span/text()").extract_first("")
        match_comment_nums = re.match(r".*?(\d+).*?", comment_nums)
        if match_comment_nums:
            comment_nums = int(match_comment_nums.group(1))
        else:
            comment_nums = 0

        content = response.xpath("//div[@class='entry']").extract_first("")

        tag_list = response.xpath('//p[@class="entry-meta-hide-on-mobile"]/a/text()').extract()
        tag_list = [element for element in tag_list if not element.strip().endswith("评论")]
        tags = ",".join(tag_list)

数据持久化

items.py创建模型

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# http://doc.scrapy.org/en/latest/topics/items.html
import datetime
import re

import scrapy
from scrapy.loader.processors import MapCompose, TakeFirst, Join
from scrapy.loader import ItemLoader


class ArticlespiderItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    pass


def date_convert(value):
    try:
        create_date = datetime.datetime.strptime(value, "%Y/%m/%d").date()
    except Exception as e:
        create_date = datetime.datetime.now().date()
    return create_date


def get_nums(value):
    match_nums = re.match(r".*?(\d+).*?", value)
    if match_nums:
        nums = int(match_nums.group(1))
    else:
        nums = 0
    return nums


def remove_comment_tags(value):
    # 去掉tags中提取的评论
    if "评论" in value:
        return ""
    else:
        return value


def return_value(value):
    return value


class ArticleItemLoader(ItemLoader):
    # 自定义itemloader
    default_output_processor = TakeFirst()


class JobBoleArticleItem(scrapy.Item):
    title = scrapy.Field()
    create_date = scrapy.Field(
        input_processor=MapCompose(date_convert)
    )
    url = scrapy.Field()
    url_object_id = scrapy.Field()
    front_image_url = scrapy.Field(
        output_processor=MapCompose(return_value)  # 覆盖掉default_output_processor
    )
    front_image_path = scrapy.Field()
    praise_nums = scrapy.Field(
        input_processor=MapCompose(get_nums)
    )
    fav_nums = scrapy.Field(
        input_processor=MapCompose(get_nums)
    )
    comment_nums = scrapy.Field(
        input_processor=MapCompose(get_nums)
    )
    content = scrapy.Field()
    tags = scrapy.Field(
        input_processor=MapCompose(remove_comment_tags),
        output_processor=Join(",")
    )

jobbole.py修改代码逻辑

class JobboleSpider(scrapy.Spider):
    name = 'jobbole'
    allowed_domains = ['blog.jobbole.com']
    start_urls = ['http://blog.jobbole.com/all-posts/']

    def parse(self, response):
        """
        1.  获取文章列表页中的文章url并交给scrapy下载后进行解析
        2.  获取下一页的url并交给scrapy进行下载，下载完成后交给parse
        """

        # 解析列表页中的所有文章url并交给scrapy下载后进行解析
        post_nodes = response.css("#archive .floated-thumb .post-thumb a")
        for post_node in post_nodes:
            image_url = post_node.css("img::attr(src)").extract_first("")
            post_url = post_node.css("::attr(href)").extract_first("")
            yield Request(url=parse.urljoin(response.url, post_url), meta={"front_image_url": image_url}, callback=self.parse_detail)

        # 提取下一页并交给scrapy下载
        next_url = response.css(".next.page-numbers::attr(href)").extract_first("")
        if next_url:
            yield Request(url=parse.urljoin(response.url, next_url), callback=self.parse)

    def parse_detail(self, response):
        article_item = JobBoleArticleItem()

        # 提取图片
        front_image_url = response.meta.get("front_image_url", "")

        # 通过item_loader加载item
        item_loader = ArticleItemLoader(item=JobBoleArticleItem(), response=response)
        item_loader.add_value("front_image_url", [front_image_url])
        item_loader.add_value("url_object_id", get_md5(response.url))
        item_loader.add_value("url", response.url)
        item_loader.add_value("front_image_url", response.url)
        item_loader.add_xpath("create_date", '//p[@class="entry-meta-hide-on-mobile"]/text()')
        item_loader.add_xpath("title", '//div[@class="entry-header"]/h1/text()')

        item_loader.add_xpath("praise_nums", "//span[contains(@class, 'vote-post-up')]/h10/text()")
        item_loader.add_xpath("fav_nums", "//span[contains(@class, 'bookmark-btn')]/text()")
        item_loader.add_xpath("comment_nums", "//a[@href='#article-comment']/span/text()")
        item_loader.add_xpath("content", "//div[@class='entry']")
        item_loader.add_xpath("tags", '//p[@class="entry-meta-hide-on-mobile"]/a/text()')

        article_item = item_loader.load_item()
        yield article_item

修改settings.py使pipeline.py生效，并设置图片自动下载

ITEM_PIPELINES = {
   'ArticleSpider.pipelines.JsonWithEncodingPipeline': 2,
   # 'scrapy.pipelines.images.ImagesPipeline': 1,
   'ArticleSpider.pipelines.ArticleImagePipeline': 1,

}
IMAGES_URLS_FIELD = "front_image_url"
project_dir = os.path.abspath(os.path.dirname(__file__))
IMAGES_STORE = os.path.join(project_dir, "images")

设置常用函数get_md5()

# -*- coding: utf-8 -*-
import hashlib


def get_md5(url):   # 不接受unicode编码
    if isinstance(url, str):
        url = url.encode("utf-8")
    m = hashlib.md5()
    m.update(url)
    return m.hexdigest()  # 完成md5的摘要生成

修改pipelines.py

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html

import codecs
import json

from scrapy.pipelines.images import ImagesPipeline
from scrapy.exporters import JsonItemExporter
from twisted.enterprise import adbapi

import MySQLdb
import MySQLdb.cursors


class ArticlespiderPipeline(object):
    def process_item(self, item, spider):
        return item


class JsonExporterPipeline(object):
    # 调用scrapy提供json_export导出json文件
    def __init__(self):
        self.file = open('articleexport.json', 'wb')
        self.exporter = JsonItemExporter(self.file, encoding='utf-8', ensure_ascii=False)
        self.exporter.start_exporting()

    def close_spider(self, spider):
        self.exporter.finish_exporting()
        self.file.close()

    def process_item(self, item, spider):
        self.exporter.export_item(item)
        return item


class MysqlTwistedPipeline(object):
    def __init__(self, dbpool):
        self.dbpool = dbpool

    @classmethod
    def from_settings(cls, settings):
        dbparams = dict(
            host=settings['MYSQL_HOST'],
            db=settings['MYSQL_DBNAME'],
            user=settings['MYSQL_USER'],
            passwd=settings['MYSQL_PASSWORD'],
            charset='utf8',
            cursorclass=MySQLdb.cursors.DictCursor,
            use_unicode=True,
        )
        # adbapi将数据化操作异步化
        dbpool = adbapi.ConnectionPool("MySQLdb", **dbparams)
        return cls(dbpool)  # 实例化

    def process_item(self, item, spider):
        #  使用twisted将MySQL插入变成异步执行
        query = self.dbpool.runInteraction(self.do_insert, item)
        query.addErrback(self.handle_error)  # 处理异常

    def handle_error(self, failure):
        # 处理异步插入的异常
        print(failure)

    def do_insert(self, cursor, item):
        # 执行具体的插入
        insert_sql = """
                    insert into jobbole_article(title, url, create_date, fav_nums)
                    values(%s, %s, %s, %s)
                """
        cursor.execute(insert_sql, (item["title"], item["url"], item["create_date"], item["fav_nums"],))


class MysqlPipeline(object):
    # 采用同步的机制写入mysql
    def __init__(self):
        self.conn = MySQLdb.connect('localhost', 'root', '', 'article_spider', charset='utf8', use_unicode=True)
        self.cursor = self.conn.cursor()

    def process_item(self, item, spider):
        insert_sql = """
            insert into jobbole_article(title, url, create_date, fav_nums)
            values(%s, %s, %s, %s)
        """
        self.cursor.execute(insert_sql, (item["title"], item["url"], item["create_date"], item["fav_nums"],))
        self.conn.commit()


class JsonWithEncodingPipeline(object):
    # 自定义json文件的导出
    def __init__(self):
        self.file = codecs.open('article.json', 'w', encoding="utf-8")

    def process_item(self, item, spider):
        lines = json.dumps(dict(item), ensure_ascii=False) + "\n"
        self.file.write(lines)
        return item

    def spider_closed(self, spider):
        self.file.closed()


class ArticleImagePipeline(ImagesPipeline):
    def item_completed(self, results, item, info):
        if "front_image_url" in item:
            for ok, value in results:
                image_file_path = value['path']
            item["front_image_url"] = image_file_path

        return item

数据库文件如items.py所示