爬取前程无忧存入数据库进行数据分析和可视化绘制词云图

爬取前程无忧网站

爬虫基本思路
获取数据在网页的位置----编写防爬-----启动数据库保存数据

本次教程使用的是mongodb数据库（MySQL）原理差不多，自己百度吧
ps：本次测试日期是2020/7/4 后面因为网站更新防爬措施无法实现功能本教程不背锅！

1、具体要求：职位名称、薪资水平、招聘单位、工作地点、工作经验、学历要求、工作内容（岗位职责）、任职要求（技能要求）。
spider文件代码

# -*- coding: utf-8 -*-
import scrapy
import re
from ..items import QianchengwuyouItem


class WuyouSpider(scrapy.Spider):
    name = 'wuyou'
    allowed_domains = ['jobs.51job.com']
    start_urls = ['http://jobs.51job.com/']

    def parse_details(selfm, response):
        # 获取详情页面数据
        print("=" * 100)
        print(response.url)
        item = QianchengwuyouItem()
        # 职位名称
        item["Job_title"] = response.xpath("//div[@class='cn']/h1/text()").extract_first()
        # 薪资水平
        item["Pay_level"] = response.xpath("//div[@class='cn']/strong/text()").extract_first()
        # 招聘单位
        item["Recruitment_unit"] = response.xpath("//div[@class='cn']//a[1]/text()").extract_first()
        # 工作地点 + 工作经验 + 学历要求....都在//div[@class='cn']/p[2]中
        item["Workplace"] = response.xpath("//div[@class='cn']/p[2]/text()[1]").get().replace('\xa0','')
        # 工作经验 + 学历要求
        all = response.xpath("//div[@class='cn']/p[2]/text()[2]").get().replace('\xa0','')
        # 判断工作经验是否存在
        if len(all) >= 4:
            item["hands_background"] = all
            item["Education_requirements"] = response.xpath("//div[@class='cn']/p[2]/text()[3]").get().replace('\xa0','')
            if len(item["Education_requirements"]) != 2:
                item["Education_requirements"] = None
        elif len(all) < 4:
            item["hands_background"] = None
            item["Education_requirements"] = all
            if len(item["Education_requirements"]) != 2:
                item["Education_requirements"] = None
            # .get().replace('\xa0','')
        # item["Workplace"] = item["Workplace"].get(1)
        # # 学历要求
        # item["Education_requirements"] = response.xpath("//div[@class='cn']/p[2]/text()[3]").get().replace('\xa0','')
        # 职位信息包含（工作内容+任职要求+工作经验+学历要求）
        item["Career_information"] = response.xpath("//div[@class='bmsg job_msg inbox']/p/text()").extract()
        item["Career_information"] = [i.strip() for i in item["Career_information"]]
        item["Career_information"] = [i for i in item["Career_information"] if len(i) > 0]
        item["Career_information"] = " ".join(item["Career_information"]).replace("\xa0","").replace(",","，")
        if (item["Pay_level"]) is None:
            item["Pay_level"] = "无"
        # 关键字：keyword
        item["keyword"] = response.xpath("//div[@class='mt10']//p//a/text()").extract()
        yield item

    def industry_perse(self, response):
        # # 获取该行业下所有职业链接
        # all_list = response.xpath("//div[@class='detlist gbox']")
        # # 获取全部招聘职位下的所有职业（occupation）链接
        # for a in all_list:
        #     occupation_url = a.xpath(".//span/a/@href").extract_first()
        #     yield scrapy.Request(
        #         occupation_url,
        #         callback=self.parse_details
        #     )
        # 获取当前页面所有职业所在的div
        all_list = response.xpath("//div[@class='detlist gbox']//div")
        # 计算当前页面获取多少url
        url_num = 0
        # 遍历获取大数据行业下的所有职业（occupation）链接
        for a in all_list:
            occupation_url = a.xpath("./p/span/a/@href").extract_first()
            yield scrapy.Request(
                occupation_url,
                callback=self.parse_details
            )
            url_num += 1
        # 翻页
        print("当前已爬取{}个职业".format(url_num))
        next_url = response.xpath("//div[@class='p_in']/ul//li/a[text()='下一页']/@href").extract_first()
        if next_url is not None:
            yield scrapy.Request(
                next_url,
                callback=self.industry_perse
            )

    def parse(self, response):
        # 获取全部招聘职位的链接
        dashujukaifa_list = response.xpath("//div[@class='maincenter']/div[2]/div[2]//a")
        # 获取全部招聘职位下的所有行业（industry）链接
        for b in dashujukaifa_list:
            industry_url = b.xpath(".//@href").extract_first()
            if industry_url == 'https://jobs.51job.com/dashujukaifa/':
                industries_url = industry_url
                yield scrapy.Request(
                    industries_url,
                    callback=self.industry_perse
                )

items.py

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/items.html

import scrapy


class QianchengwuyouItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    # 职位名称
    Job_title = scrapy.Field()

    # 薪资水平
    Pay_level = scrapy.Field()

    # 招聘单位
    Recruitment_unit = scrapy.Field()

    # 工作地点
    Workplace = scrapy.Field()

    # 工作经验
    hands_background = scrapy.Field()

    # 学历要求
    Education_requirements = scrapy.Field()

    # 职位信息（工作内容+任职要求+工作经验）
    Career_information = scrapy.Field()

    # 关键字：keyword
    keyword = scrapy.Field()

pipelines.py:启用数据库

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html

from pymongo import MongoClient



class QianchengwuyouPipeline:
    def open_spider(self,spider):
        self.db = MongoClient("localhost", 27017).qiancheng  # 创建数据库yc
        self.collection = self.db.yc_collection  # 创建一个集合

    def process_item(self, item, spider):
        #添加数据到jingjiren表中
        self.collection.insert_one(dict(item))
        return item
    def close_spider(self,spider):
        self.collection.close()

反爬在settings.py设置，看个人喜好

BOT_NAME = 'qianchengwuyou'

SPIDER_MODULES = ['qianchengwuyou.spiders']
NEWSPIDER_MODULE = 'qianchengwuyou.spiders'


# Crawl responsibly by identifying yourself (and your website) on the user-agent
USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.106 ' \
             'Safari/537.36 Edg/83.0.478.54 '

# Obey robots.txt rules
ROBOTSTXT_OBEY = False
FEED_EXPORT_ENCODING = 'utf-8'
# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32

# Configure a delay for requests for the same website (default: 0)
# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
DOWNLOAD_DELAY = 1
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default)
COOKIES_ENABLED = False
# 启用pipeline
ITEM_PIPELINES = {
     
    'qianchengwuyou.pipelines.QianchengwuyouPipeline':300,
}

mongodb效果如图

数据分析与可视化：我个人是把所有的功能都集合在一个py文件里，能注释的都注释了，大家应该能看懂

import pymongo  # python连接mongodb数据库模块
import re
from wordcloud import WordCloud  # 词云图绘制模块
from collections import Counter  # 获取数据库链接游标
from pyecharts.charts import Bar, Pie, WordCloud  # bar：条形图绘制模块，pie：饼图绘制模块，wordcloud：词云图绘制模块
from pyecharts.render import make_snapshot   # 绘图模块
from snapshot_phantomjs import snapshot
from pyecharts import options as opts, options
# pyecharts模块的详细使用教程和实例网址：http://pyecharts.org/#/

myclient = pymongo.MongoClient("127.0.0.1", port=27017)  # 数据库IP地址
mydb = myclient["qiancheng"]  # 数据库名称
mytable = mydb["yc_collection"]  # 表名称
# 分析相关岗位的平均工资、最高工资、最低工资
# def shujufenxi(query):
# 最低工资
min_salary_list = []
# 最高工资
max_salary_list = []
# 平均工资
average_salary_list = []
# 岗位
addr_list = []


class PyMongoDemo(object):

    def shujufenxi_1(self):
        Job = input("请输入你想要分析的职业：")
        # 遍历循环
        for i in mytable.find(
                {
     "$and": [
                    {
     "Job_title": {
     "$regex": "{job}".format(job=Job)}},
                    {
     "Pay_level": {
     "$regex": "万/月"}}
                ]}
            ):
            # 拆分列表获取关键数据
            min_salary = i["Pay_level"].split("-")[0]
            print(min_salary)
            max_salary = re.findall(r'([\d+\.]+)', (i["Pay_level"].split("-")[1]))[0]
            average_salary = "{:.1f}".format((float(min_salary) + float(max_salary)) / 2)
            # 设置横坐标的对象
            company = i["Job_title"]
            print(company)
            # 将获取的数据分别写入
            min_salary_list.append(min_salary)
            max_salary_list.append(max_salary)
            average_salary_list.append(average_salary)
            addr_list.append(company)
        bar = Bar(
            init_opts=opts.InitOpts(width="10000px", height="800px"),
        )
        bar.set_global_opts(
            title_opts=opts.TitleOpts(title="{}薪资".format(Job), subtitle="单位  万/月"),
            xaxis_opts=opts.AxisOpts(axislabel_opts={
     "rotate": 45}),
        )
        bar.add_xaxis(addr_list)
        bar.add_yaxis("最高薪资", max_salary_list)
        bar.add_yaxis("最低薪资", min_salary_list)
        bar.add_yaxis("平均薪资", average_salary_list)
        bar.render("{}_1.html".format(Job))

    def shujufenxi_2(self):
        Job = input("请输入你想要分析的职业：")
        for i in mytable.find(
                {
     "$and": [{
     "Job_title": {
     "$regex": "{}".format(Job)}}, {
     "Pay_level": {
     "$regex": "千/月"}}]}):
            min_salary = i["Pay_level"].split("-")[0]
            print(min_salary)
            max_salary = re.findall(r'([\d+\.]+)', (i["Pay_level"].split("-")[1]))[0]
            average_salary = "{:.1f}".format((float(min_salary) + float(max_salary)) / 2)
            company = i["Job_title"]
            print(company)
            min_salary_list.append(min_salary)
            max_salary_list.append(max_salary)
            average_salary_list.append(average_salary)
            addr_list.append(company)
        bar = Bar(
            init_opts=opts.InitOpts(width="10000px", height="800px"),
        )
        bar.set_global_opts(
            title_opts=opts.TitleOpts(title="{}薪资", subtitle="单位  千/月").format(Job),
            xaxis_opts=opts.AxisOpts(axislabel_opts={
     "rotate": 45}),
        )
        bar.add_xaxis(addr_list)
        bar.add_yaxis("最高薪资", max_salary_list)
        bar.add_yaxis("最低薪资", min_salary_list)
        bar.add_yaxis("平均薪资", average_salary_list)
        bar.render("{}_1.html").format(Job)

    def diqugangweishu(self):
        init_opts: opts.InitOpts = opts.InitOpts()
        chengdu_num = 0
        beijing_num = 0
        shanghai_num = 0
        guangzhou_num = 0
        shenzhen_num = 0
        for i in mytable.find({
     "Job_title": {
     "$regex": "大数据"}}, {
     'Job_title', 'Workplace'}):
            Workplace = i["Workplace"].split("-")[0]
            if "成都" in Workplace:
                chengdu_num += 1
            elif "北京" in i["Workplace"]:
                beijing_num += 1
            elif "上海" in i["Workplace"]:
                shanghai_num += 1
            elif "广州" in i["Workplace"]:
                guangzhou_num += 1
            elif "深圳" in i["Workplace"]:
                shenzhen_num += 1
        print(chengdu_num, beijing_num, shanghai_num, guangzhou_num, shenzhen_num)
        # all_num = chengdu_num + beijing_num + shanghai_num + guangzhou_num + shanghai_num
        data = [("成都", chengdu_num), ("北京", beijing_num), ("上海", shanghai_num), ("广州", guangzhou_num),
                ("深圳", shenzhen_num)]
        num = [chengdu_num, beijing_num, shanghai_num, guangzhou_num, shenzhen_num]
        print(data)
        # 创建条形图
        # data_pair = [list(z) for z in zip(addr, num)]
        # bar = Bar(init_opts=opts.InitOpts(width="1800px", height="800px"))
        # bar.set_global_opts(
        #     title_opts=opts.TitleOpts(title="数据分析师地区岗位个数", subtitle="单位  个"),
        #     xaxis_opts=opts.AxisOpts(axislabel_opts={"rotate": 45}),
        # )
        # bar.add_xaxis(addr)
        # bar.add_yaxis("数据分析师地区岗位个数", num)
        # bar.render("数据分析师地区岗位个数.html")

        # 创建图表对象
        pie = Pie()
        # 关联数据
        pie.add(
            # 设置系列名称
            series_name="大数据岗位地区分析",
            # 设置需要展示的数据
            data_pair=data,
            # 设置圆环空心部分和数据显示部分的比例
            radius=["30%", "70%"],
            # 设置饼是不规则的
            rosetype="radius"
        )
        # 设置数据显示的格式
        pie.set_series_opts(label_opts=options.LabelOpts(formatter="{b}: {d}%"))
        # 设置图表的标题
        pie.set_global_opts(title_opts=options.TitleOpts(title="手机销量"))
        # 数据渲染
        pie.render('数据分析地区岗位.html')

    # “数据采集”岗位要求的技能词云图
    def ciyuntu(self):
        keyword_num = 0
        a = []
        for i in mytable.find({
     }, {
     "keyword"}):
            keyword = list(i["keyword"])
            # print(keyword_list)
            a.append(keyword)
            # print(a)
            keyword_num += 1
        print(a)
        keyword_list = sum(a, [])
        word_count = {
     }
        for word in keyword_list:
            if word in word_count:
                word_count[word] += 1
            else:
                word_count[word] = 1
        print(word_count)
        lst = Counter(word_count)
        result = lst.most_common()
        # print(lst.most_common())
        print(result)
        keyword_list = ' '.join(keyword_list)

        # print(keyword_list)
        # print(keyword_num)
        def wordcloud_chart() -> WordCloud:
            c = (
                WordCloud()
                    .add("",
                         result,
                         shape="cardioid",
                         word_size_range=[20, 55], )
                    .set_global_opts(title_opts=opts.TitleOpts(title="WordCloud-shape-diamond"))
                    .render("大数据关键字词云图.html")
                # .render("大数据关键字词云图.png")
            )
            return c

        # make_snapshot(snapshot, wordcloud_chart(), "大数据关键字词云图.png")

    # 分析大数据相关岗位1-3年工作经验的薪资水平
    def fenxi1_3xinzishuiping(self):
        choice1 = "万/月"
        choice2 = "千/月"
        choice = input("请输入你要分析的薪资单位（1：万/月，2：千/月）：")
        if choice == '1':
            choice = choice1
        elif choice == '2':
            choice = choice2
        else:
            return choice
        print(choice)
        for i in mytable.find({
     
            "$or": [{
     "hands_background": {
     "$regex": "1"}}, {
     "hands_background": {
     "$regex": "2"}},
                    {
     "hands_background": {
     "$regex": "3"}}
                    ], "$and": [{
     "Job_title": {
     "$regex": "大数据"}}, {
     "Pay_level": {
     "$regex": "{}".format(choice)}}]},
                {
     "Job_title", "Pay_level", "hands_background"}):
            print(i)
            min_salary = i["Pay_level"].split("-")[0]
            # print(min_salary)
            max_salary = re.findall(r'([\d+\.]+)', (i["Pay_level"].split("-")[1]))[0]
            average_salary = "{:.1f}".format((float(min_salary) + float(max_salary)) / 2)
            company = i["Job_title"]
            # print(company)
            min_salary_list.append(min_salary)
            max_salary_list.append(max_salary)
            average_salary_list.append(average_salary)
            addr_list.append(company)
        bar = Bar(
            init_opts=opts.InitOpts(width="10000px", height="800px"),
        )
        bar.set_global_opts(
            title_opts=opts.TitleOpts(title="大数据相关岗位1-3年工作经验的薪资", subtitle="单位  {}".format(choice)),
            xaxis_opts=opts.AxisOpts(axislabel_opts={
     "rotate": 45}),
        )
        bar.add_xaxis(addr_list)
        bar.add_yaxis("最高薪资", max_salary_list)
        bar.add_yaxis("最低薪资", min_salary_list)
        bar.add_yaxis("平均薪资", average_salary_list)
        bar.render("分析大数据相关岗位1-3年工作经验的薪资水平.html")


if __name__ == "__main__":
    mongo = PyMongoDemo()
    a = 0
    b = 0
    a = str(input("请输入你要选择的功能（1：分析输入岗位的薪资水平，2：分析大数据岗位的地区分布，3：分析大数据相关岗位1-3年工作经验的薪资水平），4：绘制大数据关键字词云图："))
    while True:
        if a == '1':
            b = str(input("请输入你要选择的工资单位（1：万/月，2：千/月）："))
            if b == '1':
                mongo.shujufenxi_1()
                continue
            elif b == '2':
                mongo.shujufenxi_2()
                continue
            else:
                print("请输入正确的数字")
                continue
        elif a == '2':
            mongo.diqugangweishu()
            continue
        elif a == '3':
            mongo.fenxi1_3xinzishuiping()
            continue
        elif a == '4':
            mongo.ciyuntu()
            continue
        else:
            print("输入错误，请重新输入！")
            break
    # mongo.shujufenxi_1()
    # mongo.shujufenxi_2()
    # mongo.diqugangweishu()
    # mongo.ciyuntu()
    # mongo.fenxi1_3xinzishuiping()

该文件所使用的python模块尽量全部都安装
附上效果图



到此结束！再见各位