（新手）Python爬取智联招聘岗位信息并生成词云图

首先爬取岗位标签

image.png

1.打开岗位页

image.png

2.找到信息存放网址（代码中的"web"）

image.png

import re
import urllib.request as urlrequest
web = 'https://fe-api.zhaopin.com/c/i/sou?pageSize=90&cityId=635&salary=0,0&workExperience=-1&education=-1&companyType=-1&employmentType=-1&jobWelfareTag=-1&kw=%E4%BA%A7%E5%93%81%E7%BB%8F%E7%90%86&kt=3&=0&at=2ae1997aa6354964ac078481df22884c&rt=69444b28499d413094f94e06a6b4b161&_v=0.17798550&userCode=1036018570&x-zp-page-request-id=02072f79a0a64e619abfc5b21dc61a82-1558333624421-738392'
crawl_content = urlrequest.urlopen(web).read().decode('utf8')
#print(crawl_content)
sr=str(crawl_content)

#读取岗位页


with open("tag.txt","w",encoding='utf8') as outputfile:   #创建txt 为爬取的数据做准备
    pattern1=re.compile(r'"extractSkillTag":\[.*?]')      #正则匹配想要的内容1
    rs1=re.findall(pattern1,sr)                            
    i1=0
    for tag in rs1:                                      #输出每一个指定爬取的内容 并存档
        #print(rs1[i1])
        outputfile.write(rs1[i1]+"\n")
        i1=i1+1
        

    pattern2=re.compile(r'\\"skill\\":\[.*?]')          #正则匹配想要的内容2
    rs2=re.findall(pattern2,sr)
    i2=0
    for skill in rs2:
        #print(rs2[i2].replace("\\",""))
        outputfile.write(rs2[i2]+"\n")
        i2=i2+1
        
    pattern3=re.compile(r'"extractNormalizedTag":\[.*?]')  #正则匹配想要的内容3
    rs3=re.findall(pattern3,sr)
    i3=0
    for tag in rs3:
        #print(rs3[i3])
        outputfile.write(rs3[i3]+"\n")
        i3=i3+1
        

岗位详情页爬取岗位要求

image.png

1.找到html中内容的位置

image.png

import re
import urllib.request as urlrequest
import bs4
from bs4 import BeautifulSoup
import jieba
from wordcloud import WordCloud
import matplotlib.pyplot as plt
web = 'https://fe-api.zhaopin.com/c/i/sou?pageSize=90&cityId=635&salary=0,0&workExperience=-1&education=-1&companyType=-1&employmentType=-1&jobWelfareTag=-1&kw=%E4%BA%A7%E5%93%81%E7%BB%8F%E7%90%86&kt=3&=0&at=2ae1997aa6354964ac078481df22884c&rt=69444b28499d413094f94e06a6b4b161&_v=0.17798550&userCode=1036018570&x-zp-page-request-id=02072f79a0a64e619abfc5b21dc61a82-1558333624421-738392'
crawl_content = urlrequest.urlopen(web).read().decode('utf8')                #读取岗位页
#print(crawl_content)
p=re.compile(r'https://jobs.zhaopin.com/.*?\.htm')                             #把岗位页的详情岗位链接爬取下来
rs=p.findall(crawl_content)
with open("jobs_data.txt","w",encoding='utf-8') as outputfile:              #创建文件txt 以便存入数据
    for item in rs:                                                         #循环每一个岗位详情链接
        url=item                                                            
        wp = urlrequest.urlopen(url).read()                
        soup = BeautifulSoup(wp,'html.parser')                 
        #print(soup.prettify())
        rq=soup.find(class_="describtion__detail-content").get_text()      #解析html 然后找到想要内容的位置
        #print(rq)
        p = re.compile(r'岗位要求：.*|岗位职责：.*|.*要求：.*|职位描述：.*')  #匹配想要的内容
        pp=p.findall(rq)
        pa = re.compile(r'【你要做的工作】.*')
        pa1 = pa.findall(rq)
        if pa1:
            pa2=pa1[0]
            #print(pa2)
            #print(" ")
            #print("******找到岗位哪里的信息，并且变为字符串*******")
            #print(" ")

            pattern = re.compile(r'【我们想要的你】')
            pp2=pattern.sub('\n【我们想要的你】',pa2)
            #print(pp2)
            #print(" ")
            #print("******分隔任职和岗位***********")
            #print(" ")    

            pattern = re.compile(r'【我们给你的】.*')
            pp2=pattern.sub('',pp2)
            #print(pp2)
            #print("***********去除多余的*********")

            pattern = re.compile(r'\d、|\d\.')
            bb=pattern.sub('\n\t - ',pp2)
            #print(bb)
            #print(" ")
            #print("******把所有的数字符号隔行美观***")
            #print(" ")
            outputfile.write(bb+"\n")
        if pp:
            pp1=pp[0]
            #print(pp1)
            #print(" ")
           # print("******找到岗位哪里的信息 并且变为字符串*******")
            #print(" ")

            pattern = re.compile(r'任职要求：|任职资格：|岗位职责：')
            pp2=pattern.sub('\n任职要求：',pp1)
           # print(pp2)
           # print(" ")
           # print("******分隔任职和岗位***********")
           # print(" ")

            pattern = re.compile(r'\d、|\d\.')
            bb=pattern.sub('\n\t - ',pp2)
            #print(bb)
            #print(" ")
            #print("******把所有的数字符号隔行美观***")
            #print(" ")
            outputfile.write(bb+"\n")
        else:
                ## 没有文字提示的时候    
            p1 = re.compile(r'\d、|\d.')
            bb=pattern.sub('\n\t - ',rq)
            #print(bb)
            outputfile.write(bb+"\n")

生成词云图

image.png

import jieba
from wordcloud import WordCloud
import matplotlib.pyplot as plt
file=open("tag.txt",encoding='utf-8').read()           #打开读取文档
cw=jieba.cut(file)                                     #分词
result = " ".join(cw)                                  #把词语以空格连接在一起
wc = WordCloud(                                     #生成云图
    font_path = "simhei.ttf",
    background_color='white',   #背景颜色
    width=1000,
    height=1000,
    max_words=500,
    stopwords=["经理","产品","extractSkillTag","skill","extractNormalizedTag"]
)
wc.generate(result)
wc.to_file('tag_ph.png')    #图片保存

#5.显示图片
plt.figure('tag_ph')   #图片显示的名字
plt.imshow(wc)  
#plt.imshow()函数负责对图像进行处理，并显示其格式，作用：将一个image显示在二维坐标轴上。  
#其后跟着plt.show()才能显示出来。
plt.axis('off')        #关闭坐标
plt.show()