scrapy_redis 爬取58电销招聘信息

环境

• win10
• Pycharm
• Python 3.6.1
• Scrapy 1.0
• scrapy_redis
• mysql5.7

思路

• 爬取某个城市（以杭州为例）各个行业电销的招聘信息
• 爬虫分为两部分
  • 生产者（industry_spider.py）：爬取各行业电话销售的url， 以集合的方式存储到redis中
  • 消费者（recruitment_spider.py）: 从redis中提取url, 作为初始url。然后爬取详细信息。
• 信息存储到mysql中

步骤

• 数据表创建

DROP TABLE IF EXISTS `recruitment_info`;

CREATE TABLE `recruitment_info(
`id` int(11) unsigned not null auto_increment,
`title` varvahr(128) not null,
`salary` varchar(64) not null,
`company` varchar(128) not null,
`website` varcahr(128) not null,
primary key `id`,
unique key `url` (`website`)
);

• 创建项目：
  • scrapy startproject recruitment_58
• 构建爬虫：
  • scrapy genspider industry_spider hz.58.com/job.shtml
  • scrapy genspider recruitment_spider hz.58.com/dianhuaxiaoshou
• settings.py

# -*- coding: utf-8 -*-

BOT_NAME = 'recruitment_58'

SPIDER_MODULES = ['recruitment_58.spiders']
NEWSPIDER_MODULE = 'recruitment_58.spiders'

# Crawl responsibly by identifying yourself (and your website) on the user-agent
USER_AGENTS = [
    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)",
    "Mozilla/4.0 (compatible; MSIE 7.0; AOL 9.5; AOLBuild 4337.35; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
    "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)",
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
    "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
    "Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",
    "Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0",
    "Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5",
    "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20",
    "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)",
    "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1",
    "Mozilla/5.0 (iPad; U; CPU OS 4_2_1 like Mac OS X; zh-cn) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8C148 Safari/6533.18.5",
    "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:2.0b13pre) Gecko/20110307 Firefox/4.0b13pre",
    "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:16.0) Gecko/20100101 Firefox/16.0",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11",
    "Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
]

# Obey robots.txt rules
ROBOTSTXT_OBEY = False

# Configure maximum concurrent requests performed by Scrapy (default: 16)
CONCURRENT_REQUESTS = 32

# Configure a delay for requests for the same website (default: 0)
# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
DOWNLOAD_DELAY = 1

# Disable cookies (enabled by default)
COOKIES_ENABLED = False

# Disable Telnet Console (enabled by default)
# TELNETCONSOLE_ENABLED = False

# Override the default request headers:
DEFAULT_REQUEST_HEADERS = {
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
    'Accept-Encoding': 'gzip, deflate',
    'Accept-Language': 'zh-CN,zh;q=0.9',
    'Cache-Control': 'max-age=0',
}

DOWNLOADER_MIDDLEWARES = {
    # 'Top250.middlewares.Top250DownloaderMiddleware': None,
    'recruitment_58.middlewares.RandomUserAgentMiddleware': 300
}

# Configure item pipelines
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
    'recruitment_58.pipelines.Recruitment58Pipeline': 300,
}

# database
MYSQL_HOST = 'localhost'
MYSQL_DB = 'corporate_info'
MYSQL_USER = 'root'
MYSQL_PASSWD = '654321'

# Redis
REDIS_HOST = '127.0.0.1'
REDIS_PORT = 6379
REDIS_DB = 0

# 必须1：表示使用scrapy-redis提供的去重类，也就是在Redis数据库中去重
DUPEFILTER_CLASS = "scrapy_redis.dupefilter.RFPDupeFilter"

# 必须2：表示使用scrapy-redis提供的调度器类，也就是和Redis数据库交互请求数据
SCHEDULER = "scrapy_redis.scheduler.Scheduler"

# 必须3：表示程序可以中途暂停，不清空Redis的请求队列
SCHEDULER_PERSIST = True

• items.py

from scrapy import Field, Item


class IndustryUrlItem(Item):
    url = Field()   # 行业电销招聘网址


class Recruitment58Item(Item):
    # define the fields for your item here like:
    title = Field()     # 招聘标题
    salary = Field()    # 薪水
    company = Field()   # 公司名称
    website = Field()   # 公司简介网址

• industry_spider.py

# -*- coding: utf-8 -*-
import scrapy
from ..items import IndustryUrlItem


class IndustrySpiderSpider(scrapy.Spider):
    name = 'industry_spider'
    allowed_domains = ['hz.58.com']
    start_urls = ['http://hz.58.com/job.shtml']
    custom_settings = {
        'ITEM_PIPELINES': {
            'phone58.pipelines.RedisStartUrlsPipeline': 301,
        },
        'DOWNLOADER_MIDDLEWARES': {
          # 'phone58.middlewares.ProxyMiddleware': 543
        },
    }

    def parse(self, response):
        """
        采集各个行业的电话销售链接
        :param response:
        :return:
        """
        base_url = 'http://hz.58.com/dianhuaxiaoshou'
        for data in response.xpath('//*[@id="divIndCate"]/ul/li[position()>1]'):
            item = IndustryUrlItem()
            suffix_url = data.xpath('./a/@href').extract_first()
            url = base_url + suffix_url
            item['url'] = url
            yield item

• recruitment_spider.py
  • REDIS_START_URLS_AS_SET : 默认为False
  • 使用集合操作lpop()从redis队列中取出url, 取出后集合中就没有了那个url

# -*- coding: utf-8 -*-
from scrapy_redis.spiders import RedisSpider
from ..items import Recruitment58Item


class PhoneSpiderSpider(RedisSpider):
    name = 'recruitment_spider'
    redis_key = "industry:start_urls"

    custom_settings = {
        'REDIS_START_URLS_AS_SET': True,
    }

    def parse(self, response):
        self.logger("Starting Crawl %s" % response.url)
        resp = response.xpath('//*[@id="list_con"]/li')
        for i in resp:
            item = Recruitment58Item()
            title = i.xpath('.//div[@class="job_name clearfix"]/a/span/text()').extract()
            item['title'] = title[0] + '|' + title[-1]
            item['salary'] = i.xpath('.//p/text()').extract_first()
            item['company'] = i.xpath('./div[@class="item_con job_comp"]/div/a/text()').extract_first()
            item['website'] = i.xpath('.//div[@class="comp_name"]/a/@href').extract_first()
            yield item
        next_page = response.xpath('//div[@class="pagesout"]/a[@class="next"]/@href').extract_first()
        if next_page is not None:
            self.logger.info("Start Crawl: %s" % next_page)
            yield response.follow(next_page, callback=self.parse)

• pipelines.py

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
import redis
import MySQLdb
import MySQLdb.cursors
from twisted.enterprise import adbapi


class RedisDistrictUrlsPipeline(object):
    """
    按行业提取电话销售url存到redis中
    """
    def __init__(self, host, port, db):
        self.redis_client = redis.StrictRedis(
            host=host, port=port, db=db
        )

    @classmethod
    def from_crawler(cls, crawler):
        return cls(
            host=crawler.settings.get("REDIS_HOST"),
            port=crawler.settings.get("REDIS_PORT"),
            db=crawler.settings.get("REDIS_DB"),
        )

    def process_item(self, item, spider):
        redis_key = 'industry:start_urls'
        url = item['url']
        if url:
            self.redis_client.sadd(redis_key, url)
            spider.logger.debug(
                '****** Success push to REDIS with {} ******'.format(url))
            return item


# 对数据库异步操作
class Recruitment58Pipeline(object):

    def __init__(self, dbpool):
        self.dbpool = dbpool

    @classmethod
    def from_settings(cls, settings):
        dbparms = dict(
            host=settings.get("MYSQL_HOST"),
            db=settings.get("MYSQL_DB"),
            user=settings.get("MYSQL_USER"),
            passwd=settings.get("MYSQL_PASSWD"),
            charset='utf8',
            use_unicode=True,
        )
        # adbapi.ConnectionPool构造器，进行数据库的异步操作语句
        dbpool = adbapi.ConnectionPool("MySQLdb", **dbparms)
        # 返回取得的值
        return cls(dbpool)

    # 使用twistd异步插入
    def process_item(self, item, spider):
        query = self.dbpool.runIneraction(self.do_insert, item)
        query.addErrback(self.handle_error)

    # 处理异步插入异常
    def handle_error(self, failure):
        print(failure)

    # 执行具体的插入
    def do_insert(self, cursor, item):
        insert_sql = """
                    insert into recruitment_info(title,salary,company,website) 
                    values (%s,%s,%s,%s)"""
        cursor.execute(insert_sql, (item['title'], item['salary'], item['company'], item['website']))

• 数据截图

数据截图.png

• 源码参考