爬取招聘网站信息，并使用pyecharts和matplotlib进行简单的可视化测试
首先我们先来爬取网站获取所需要的信息，我选择的是boss直聘，别问，问就是拉勾网难爬。
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36',
'cookie':'_uab_collina=155504712754406507756888; lastCity=101010100; _uab_collina=155514097560048373468469; sid=sem_pz_bdpc_dasou_title; __c=1566816316; __g=sem_pz_bdpc_dasou_title; Hm_lvt_194df3105ad7148dcf2b98a91b5e727a=1566816317; __l=l=%2Fwww.zhipin.com%2F%3Fsid%3Dsem_pz_bdpc_dasou_title&r=https%3A%2F%2Fsp0.baidu.com%2F9q9JcDHa2gU2pMbgoY3K%2Fadrc.php%3Ft%3D06KL00c00fDIFkY0IWPB0KZEgsDv8API00000Kd7ZNC00000I05PWc.THdBULP1doZA80K85yF9pywd0ZnqrAm3PjwBPjRsnj0sn10vm6Kd5HP7rRDkPRN7nj9Kn1b3PbuKwj-awjRvnbmswHTzwWfs0ADqI1YhUyPGujY1nWcYPW0Yn1f3FMKzUvwGujYkP6K-5y9YIZK1rBtEILILQMGCpgKGUB4WUvYE5LPGujd1uydxTZGxmhwsmdqbmgPEINqYpgw_ufKWThnqPj6dPjc%26tpl%3Dtpl_11534_19713_15764%26l%3D1511867677%26attach%3Dlocation%253D%2526linkName%253D%2525E6%2525A0%252587%2525E5%252587%252586%2525E5%2525A4%2525B4%2525E9%252583%2525A8-%2525E6%2525A0%252587%2525E9%2525A2%252598-%2525E4%2525B8%2525BB%2525E6%2525A0%252587%2525E9%2525A2%252598%2526linkText%253DBoss%2525E7%25259B%2525B4%2525E8%252581%252598%2525E2%252580%252594%2525E2%252580%252594%2525E6%252589%2525BE%2525E5%2525B7%2525A5%2525E4%2525BD%25259C%2525EF%2525BC%25258C%2525E6%252588%252591%2525E8%2525A6%252581%2525E8%2525B7%25259F%2525E8%252580%252581%2525E6%25259D%2525BF%2525E8%2525B0%252588%2525EF%2525BC%252581%2526xp%253Did(%252522m3224604348_canvas%252522)%25252FDIV%25255B1%25255D%25252FDIV%25255B1%25255D%25252FDIV%25255B1%25255D%25252FDIV%25255B1%25255D%25252FDIV%25255B1%25255D%25252FH2%25255B1%25255D%25252FA%25255B1%25255D%2526linkType%253D%2526checksum%253D8%26ie%3DUTF-8%26f%3D8%26tn%3Dbaidu%26wd%3Dboss%25E7%259B%25B4%25E8%2581%2598%26oq%3Dboss%25E7%259B%25B4%25E8%2581%2598%26rqlang%3Dcn&g=%2Fwww.zhipin.com%2F%3Fsid%3Dsem_pz_bdpc_dasou_title&friend_source=0&friend_source=0; __zp_stoken__=ad34ePUaLiUVfgZxwD%2FUM1iMttFjglIo6h5fCIBYoYizouEqwp9CPryEab7aReT9hp7bYfRKmvHTBx2nRg6vj7g4Og%3D%3D; Hm_lpvt_194df3105ad7148dcf2b98a91b5e727a=1566816407; __a=45336496.1555046589.1556256159.1566816316.43.9.5.5'
}
dic = []
city = ['c101020100','c101010100','c101210100']
for j in city:
for i in range(1,4):
url = 'https://www.zhipin.com/{0}/?query=python&page={1}'.format(j,i)
dic.append(url)
老规矩，先对请求的地址进行获取，cookie需要加，不加提取不到数据。我需要的是上海北京杭州的各90条数据，下面是对网页信息的选择，如图
def getmasesage(n):
response = requests.get(url=n,headers=headers)
response.encoding='utf-8'
# print(response.text)
select = Selector(text=response.text)
data = select.css('.job-list ul li')
# print(data)
listda = []
for i in data:
dict = {}
dict['职位'] = i.css('.info-primary .job-title ::text').extract()[0]
dict['薪资'] = i.css('.info-primary .red ::text').extract()[0]
dict['地区'] = i.css('.info-primary p ::text').extract()[0]
dict['工作经验'] = i.css('.info-primary p ::text').extract()[1]
dict['学历'] = i.css('.info-primary p ::text').extract()[2]
dict['公司名称'] = i.css('.company-text .name a ::text').extract()[0]
listda.append(dict)
time.sleep(3)
return listda
数据提取完成后我们将它使用pandas存入到excel以便于下次处理数据
data_lst = []
n = 1
for j in dic:
data_lst.extend(getmasesage(j))
print('成功采集{}条数据'.format(n * 30))
n += 1
df_result = pd.DataFrame(data_lst)
print(df_result)
df_result.to_excel(r'boss直聘.xlsx')#导入到excel文件
结果如图
获得数据以后我们就可以对其进行分析啦
下面是简单的对于pandas的一些应用，每条都有注释啦！
import pandas as pd
df = pd.read_excel('boss直聘.xlsx')
print(df[df['学历']=='本科'])#筛选学历为本科的数据
print(df.shape[0])#查看列数
print(df.describe())#查看简要信息
df['最低薪资'] = df['薪资'].str.split('-').str.get(0).astype(int)#举个数据清洗的例子给出最低薪资
print(df)#查看结果会多个新的一列
print(df['最低薪资'].mean())#计算最低薪资的平均值
data=df[-df['工作经验'].isin(['经验不限'])]#去掉数据表中经验不限的数据
print(data['工作经验'])#检验确实没有了经验不限数据
部分结果如图
接下来是对我们所需要的数据进行处理，以便于之后画图
m = df[['学历','工作经验','最低薪资']]#将df中的三列数据转为另一个新的数据集
print(m[m.最低薪资>15])#筛选最低薪资大于15k的
op =m[m.学历=='本科']['最低薪资']#列出学历为本科的最低薪资
op1 =m[m.学历=='大专']['最低薪资']#列出学历为本科的最低薪资
op2 =m[m.学历=='硕士']['最低薪资']#列出学历为本科的最低薪资
print(op.mean())#计算平均值
p1 = len(df[(df.学历=='本科')&(df.最低薪资>15)&(df.工作经验=="经验不限")])#根据要求提取出满足条件的人数
p2 =len(df[(df.学历=='本科')&(df.最低薪资>15)&(df.工作经验=="1-3年")])
p3 =len(df[(df.学历=='本科')&(df.最低薪资>15)&(df.工作经验=="3-5年")])
p4 =len(df[(df.学历=='本科')&(df.最低薪资>15)&(df.工作经验=="5-10年")])
p5 =len(df[(df.学历=='本科')&(df.最低薪资>15)&(df.工作经验=="应届生")])
p = [p1,p2,p3,p4,p5]
label = ['经验不限','1-3年','3-5年','5-10年','应届生']
h = len(m[m.学历=='本科'])/len(m)#计算各学历的人数占企业需求的百分比
h1 = len(m[m.学历=='大专'])/len(m)
h2 = len(m[m.学历=='硕士'])/len(m)
h3 = (len(m)-len(m[m.学历=='硕士'])-len(m[m.学历=='大专'])-len(m[m.学历=='本科']))/len(m)
部分结果
数据提取完毕后，我们就可以开始画图啦
以下是普通柱状图与饼图的绘制
import matplotlib as plt
from pylab import *
#设置字体，不舍之中文会出不来
mpl.rcParams['font.sans-serif'] = ['SimHei']
plt.figure(figsize=(12,5))
#标题
plt.title("薪资与学历的关系")
dc = ['本科','大专','硕士']
dy = [op.mean(),op1.mean(),op2.mean()]
plt.bar(dc,dy,align = 'center',color='red',alpha=0.8,width=0.3)#画出柱状图
plt.xlabel('工作经验')
plt.ylabel('薪资水平')
plt.show()
plt.title("企业对于学历的需求分布")
print(h,h1,h2,h3)
labels = ['本科','大专','硕士','其他']
oooooo = [h,h1,h2,h3]
tu = [0.5,0,0,0]
plt.pie(oooooo,labels=labels,shadow=True,explode=tuple(tu),autopct='%1.1f%%')#绘制饼图，并将最大的一块突出显示
plt.show()
效果图
有好多细节也能再添加让图更好看，但是没做，别问，问就是懒
然后是使用pyecharts制作的3D柱状图与交互式饼图
from pyecharts import options as opts
from pyecharts.charts import Bar3D
from pyecharts.charts import Pie
data_xyz=[]
#3D柱状图
def bar3d_base() -> Bar3D:
for i in range(50):
x = m.iloc[i, 0]
y = m.iloc[i, 1]
z = m.iloc[i, 2]
print(z)
data_xyz.append([x, y, z])
c = (
Bar3D()
.add(
"",
[ [d[0], d[1], str(d[2])] for d in data_xyz],
xaxis3d_opts=opts.Axis3DOpts(df['学历'], type_="category"),
yaxis3d_opts=opts.Axis3DOpts(df['工作经验'], type_="category"),
zaxis3d_opts=opts.Axis3DOpts(df['最低薪资'], type_="category"),
)
.set_global_opts(
visualmap_opts=opts.VisualMapOpts(max_=20),
title_opts=opts.TitleOpts(title="boss"),
)
)
print(data_xyz)
return c.render("bbb.html")
bar3d_base()
其实画的道理就是这个理，很多细节上的处理比如提示或者修改什么的，参考下官方文档很容易！，然后是饼图
def pie_base() -> Pie:
c = (
Pie()
.add("", [list(z) for z in zip(label, p)], center=(500, 280),radius=["40%", "75%"])
.set_global_opts(title_opts=opts.TitleOpts(title="各个工作阶段薪资大于15k的本科人数"),legend_opts=opts.LegendOpts(orient="scroll", pos_left="1%", pos_top="25%"))
.set_series_opts(label_opts=opts.LabelOpts(formatter="{b}: {c}"))
)
return c.render("bbbh.html")
pie_base()
我觉得pyecharts真的超级棒！交互式的图表可以给人更直观的感受，我只是举了两个例子，有各种可交互式的图表可以绘制，什么旭日图，河流图，漏斗图，雷达图，各地的地图之类，给个参考！