智联岗位爬虫，半自动化

参考文档：

python selenium实现智联招聘数据爬取python脚本之家 (jb51.net)

安装selenium 和chromeDriver文档：

selenium 安装与 chromedriver安装 - Rogn - 博客园 (cnblogs.com)

注意：

· 因为页面存在动态加载，通过调整页面大小规避

· 智联页面无法自动输入账号密码，会被判断用户名错误，所以要手动输入，前提保证睡眠时间够用，可适当加长

· 此代码需要在个人电脑调试一下，并不是万能的，cookie ，页面大小，驱动地址都需要更改调试

代码：

# -*- coding: utf-8 -*-

"""

Created on Mon Oct 11 19:14:00 2021

@author:

"""

from urllib.request import quote, unquote

import csv

import os

import re

import json

import time

import requests

from selenium.webdriver import Chrome

from selenium.webdriver.remote.webelement import WebElement

from selenium.webdriver.common.by import By

from selenium.webdriver.support import ui

from selenium.webdriver.support import expected_conditions

from lxml import etree

#驱动地址

chrome =

Chrome(executable_path=r'C:\Users\XX\AppData\Local\Google\Chrome\Application\chromedriver.exe')

# 因为页面存在动态加载，适当更改页面比例，规避动态显示

chrome.maximize_window()

chrome.execute_script("document.body.style.zoom='0.75'") #缩小

# 模拟登录

def login(url):

 # 获取登录页面

 chrome.get(url)

 ##下面代码自动输入，会判断密码错误，需要手动输入，点击登录，滑块

 ##如果时间不够可以令睡眠时间长一点

 time.sleep(20)

 get_allcity('https://www.zhaopin.com/citymap')

 # 在登录状态下进行所有城市信息的获取

def get_allcity(url):

 resp = requests.get(url, headers=headers)

 if resp.status_code == 200:

 html = resp.text

 json_data = re.search(r'', html).groups()[0]

 data = json.loads(json_data)

 cityMapList = data['cityList']['cityMapList'] # dict

 for letter, citys in cityMapList.items():

 for city in citys: # citys 是个列表，里面嵌套的字典

 city_name = city['name']

 city_url = 'https:' + city['url']

 # 筛选城市

 query_citys = ('成都')

 if city_name in query_citys:

 print(f'正在获取{city_name}的信息')

 get_city_job(city_url)

 time.sleep(3)

 else:

 # print(f'{city_name} 不在搜索范围内！')

 pass

 else:

 print('网页获取失败')

def get_city_job(url):

 chrome.get(url) # 打开城市信息

 # 根据class_name 查询WebElement找出输入的位置

input_seek: WebElement = chrome.find_element_by_class_name('zp-search__input')

#输入查询岗位

input_seek.send_keys('Python')

# 找出搜索按钮并点击

click: WebElement =

chrome.find_element_by_xpath('//div[@class="zp-search__common"]//a') 

 click.click()

 # 切换到第二个页面

 chrome.switch_to.window(chrome.window_handles[1])

 time.sleep(1)

 time.sleep(1)

 # 等待class_name为“sou-main__list” div元素出现

 ui.WebDriverWait(chrome, 30).until(

 expected_conditions.visibility_of_all_elements_located((By.CLASS_NAME, 'sou-main__list')),

 '查找的元素一直没有出现'

 )

 # 判断当前查询结果是否不存在

 no_content = chrome.find_elements_by_class_name('positionlist')

 if not no_content:

 print('当前城市未查找到Python岗位')

 else:

 # 提取查找结果

 parse(chrome.page_source)

def parse(html):

 root = etree.HTML(html)

 divs = root.xpath('//div[@class="positionlist"]') # element对象

 items = {}

 for div in divs:

 # 岗位

 position = div.xpath('.//a//div[@class="iteminfo__line1__jobname"]/span[1]')

 # 公司

 company = div.xpath('//a//div[@class="iteminfo__line1__compname"]/span/text()')

 # 薪资

 money = div.xpath('.//a//div[@class="iteminfo__line2__jobdesc"]/p/text()')

 # 位置

 city = div.xpath('//a//div[@class="iteminfo__line2__jobdesc"]/ul/li[1]/text()')

 # 经验

 experience = div.xpath('.//a//div[@class="iteminfo__line2__jobdesc"]/ul/li[2]/text()')

 # 学历

 education = div.xpath('.//a//div[@class="iteminfo__line2__jobdesc"]/ul/li[3]/text()')

 # 规模

 scale = div.xpath('.//a//div[@class="iteminfo__line2__compdesc"]/span[1]/text()')

 # 人数

 people = div.xpath('.//a//div[@class="iteminfo__line2__compdesc"]/span[2]/text()')

 for position_, company_, money_, city_, experience_, education_, scale_, people_ in zip(position, company,

 money, city, experience,

 education, scale,

 people):

 # title="python爬虫工程师" 获取它的title属性值

 string = position_.attrib.get('title')

 items['position'] = string

 items['company'] = company_

 items['money'] = money_.strip()

 items['city'] = city_

 items['experience'] = experience_

 items['education'] = education_

 items['scale'] = scale_

 items['people'] = people_

 itempipeline(items)

 #print(items)

 # 获取下一页

 next_page()

def itempipeline(items):

 has_header = os.path.exists(save_csv) # 文件头

 with open(save_csv, 'a', encoding='utf-8-sig') as file:

 writer = csv.DictWriter(file, fieldnames=items.keys())

 if not has_header:

 writer.writeheader() # 写入文件头

 writer.writerow(items)

def next_page():

 # 找到下一页按钮

 time.sleep(1)

 # 页面有动态加载无法直接从顶部开始拉滚动条，所以选择底部

 js="window.scrollTo(document.body.scollHeight,100)"

 chrome.execute_script(js)

 #找到下一页按钮

 button = chrome.find_elements_by_xpath('/div[@class="soupager"]/button[@class="btn soupager__btn"]')

 if not button:

 print(f'获取完毕，请在 {save_csv} 里查看!!')

 exit()

 else:

 time.sleep(1)

 button[0].click() # 点击下一页

 #chrome.execute_script("arguments[0].click();", button)

 time.sleep(3)

 parse(chrome.page_source)

if __name__ == '__main__':

 n = 0

 ####cookies 是网址“https://www.zhaopin.com/citymap” 这一页的

 cookies ='"x-zp-client-id=f1e50b3c-66e1-4026-b065-e91d60707ea6; sajssdk_2015_cross_new_user=1; sensorsdata2015jssdkcross={"distinct_id":"17c706bcc1754f-0f0b96a673e2808-b7a1438-1024000-17c706bcc18593","first_id":"","props":{"$latest_traffic_source_type":"直接流量","$latest_search_keyword":"未取到值_直接打开","$latest_referrer":""},"$device_id":"17c706bcc1754f-0f0b96a673e2808-b7a1438-1024000-17c706bcc18593"}; zp_passport_deepknow_sessionId=bd94ba2bs00e1943148a588024abfb073d03; acw_tc=2760826216339736981194712e27e6f8af94db9c58e502ce71ac5d5b427b8e; sts_deviceid=17c706bfdd1365-045fa83fea5838-b7a1438-1024000-17c706bfdd218; LastCity_id=801; LastCity=成都; sts_sg=1; sts_sid=17c706bfedc683-071b312edb9e43-b7a1438-1024000-17c706bfedd817; sts_chnlsid=Unknown; ZL_REPORT_GLOBAL={"//www":{"seid":"","actionid":"932b2668-11af-450d-a229-f10072031ebc-cityPage"}}; Hm_lvt_363368edd8b243d3ad4afde198719c4a=1633973699; 1420ba6bb40c9512e9642a1f8c243891=5930d939-a8aa-4835-af18-1b800a782a12; locationInfo_search={"code":"606","name":"营口","message":"匹配到市级编码"}; Hm_lvt_38ba284938d5eddca645bb5e02a02006=1633973701; Hm_lpvt_38ba284938d5eddca645bb5e02a02006=1633973701; Hm_lpvt_363368edd8b243d3ad4afde198719c4a=1633973947; sts_evtseq=7"'

 headers = {

 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36',

 'Cookie': cookies.encode('utf-8').decode('latin-1'),

 'content-type':'text/html; charset=utf-8'

 }

 save_csv = 'chengdu-python.csv'

 login(

 "https://passport.zhaopin.com/login?bkUrl=%2F%2Fi.zhaopin.com%2Fblank%3Fhttps%3A%2F%2Fsou.zhaopin.com%2F%3Fjl%3D801%26kw%3DPython%26kt%3D3%26srccode%3D")

结果

结果