Python爬取BOSS直聘商业分析岗职位描述信息

1 爬虫工具及思路

1.工具包： selenium-webdriver 模拟浏览器（因为网页是动态加载的，而且我比较难以找到json包）
2.思路： 首先获取每个岗位的id，然后根据id构造岗位url，最后逐一访问网址并抓取所需信息。

2 准备工作

1. python中下载selenium包，pip install selenium
2. webdriver 官网下载对应浏览器的webdriver驱动，一定要和原文件在同一个目录下，附上谷歌官网网址：谷歌浏览器webdriver下载地址
3. selenium模拟浏览器的相关操作可以参考这篇文章：python selenium库的使用
4. 正则表达式相关操作参照下面这篇博客python正则表达式的应用

3 代码

from selenium import webdriver
import re
import xlwt
import time
# define method that will create a browser, suitable to your operating system
import sys
def get_a_browser():
    if sys.platform.startswith('win32') or sys.platform.startswith('cygwin'):
        return webdriver.Chrome() # windows
    else:
        return webdriver.Chrome('./chromedriver') # mac
#get job id
job_id = []
part = 'data-jid="(.*?)"'
url = u'https://www.zhipin.com/c100010000/?query=%E5%95%86%E4%B8%9A%E5%88%86%E6%9E%90%E5%B8%88&page=9&ka=page-next'
browser = get_a_browser()
browser.get(url)
time.sleep(1)
body = browser.find_element_by_tag_name('body')
for i in range(0,15):
    soup = BeautifulSoup(browser.page_source, 'html.parser')
    info = re.findall(part,str(soup))
    job_id.extend(info)
    browser.find_element_by_class_name('next').click()
    time.sleep(3)
#获取搜索职位的id
job_id = []
part = 'data-jid="(.*?)"'
url = u'https://www.zhipin.com/c100010000/?query=%E5%95%86%E4%B8%9A%E5%88%86%E6%9E%90%E5%B8%88&page=9&ka=page-next'
browser = get_a_browser()
browser.get(url)
time.sleep(1)
body = browser.find_element_by_tag_name('body')
for i in range(0,15):
    soup = BeautifulSoup(browser.page_source, 'html.parser')
    info = re.findall(part,str(soup))
    job_id.extend(info)
    browser.find_element_by_class_name('next').click()#根据class named定位页面下方的next按钮位置并点击进入下一页
    time.sleep(3)
#根据job_id访问每一个职位的网页并获取相关信息，最后存储在Excel表格中
book = xlwt.Workbook(encoding='utf-8') 
sheet1 = book.add_sheet(u'Sheet1', cell_overwrite_ok=False)
try : 
    for i in range(0,len(job_id)):
        job_url = 'https://www.zhipin.com/job_detail/{}.html'.format(new_job_id[i])
        browser.get(job_url)
        data = str(browser.page_source)
        pat1 = '
(.*?)
'
        job_title = re.compile(pat1).findall(data)[0]
        pat2 = '
(.*?)(.*?)(.*?)
'
        info = re.compile(pat2).findall(data)
        city = info[0][0]
        work_experience  = info[0][1]
        education = info[0][2]
        pat3 = '(.*?)'
        salary = re.compile(pat3).findall(data)[0]
        pat4 = '
职位描述
[\s\S]*?
([\s\S]*?)
'
        job_description = re.compile(pat4).findall(data)[0].strip().replace('
','')
        pat5 = ' 
([\s\S]*?)
        company = re.compile(pat5).findall(str(data))[0].strip()
        sheet1.write(i, 0, job_title)  
        sheet1.write(i, 1, company)
        sheet1.write(i, 2, city)       
        sheet1.write(i, 3, work_experience)       
        sheet1.write(i, 4, education)       
        sheet1.write(i, 5, salary)
        sheet1.write(i, 6, job_description)
    book.save('E:\postgraduate study\jupyter\job_description_nation.xls')
except Exception as e:
    book.save('E:\postgraduate study\jupyter\job_description_natioin.xls')
    print(i)

4 运行结果