GitHub链接：
BossSpider
值得体味的几个点：
- 可以使用json获得列表中的职位detail数据
- 创建类可以更好的携带proxy、headers等属性
- 使用代理避开IP封锁和验证码
- 理清楚逻辑很重要，对于异常处理
import time
import requests
from requests import Session
from http import client
import re
from lxml import etree
import json
import random
client._is_legal_header_name = re.compile(br':|\A[^:\s][^:\r\n]*\Z').match
class BossSpider():
session = Session()
num = 0
company ={
# 'zhifubao': '1f8227007663695b3nV63Q~~',
# 'baidu': 'ab9fdc6f043679990HY~',
# 'tenxun': '2e64a887a110ea9f1nRz',
# 'ali': '5d627415a46b4a750nJ9',
# 'zijietiaodong': 'a67b361452e384e71XV82N4~',
# 'huawei':'02cd05cce753437e33V50w~~',
# 'wangyi':'821662f4a993420c3nZ62dg~',
'jindong':'755f9f2f1799f89a03d60g~~',
'mayi':'e9a2427bc19f1b1a33Z_3t0~',
'jdjituan':'33e052361693f8371nF-3d25',
'BJzijie':'7d01ce6cfe2022030HJ409y-',
'bjtenxun':'0979070fc97e501a0nx7'
}
USER_AGENT_LIST=[
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1",
"Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6",
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5",
"Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24",
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24"
]
def __init__(self):
self.proxy = self.get_proxy()
self.agent = random.choice(self.USER_AGENT_LIST)
self.headers = {
':authority': 'www.zhipin.com',
':method': 'GET',
# ':path': urlb,
':scheme': 'https',
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
'accept-encoding': 'gzip, deflate, br',
'accept-language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',
'cache-control': 'max-age=0',
# 'referer': urla + '/{}/?page={}&ka=page-{}'.format(cp,page-1,page-1) if page>1 else 'https://www.zhipin.com/c101020100-p100109/',
'user-agent': self.agent,
}
def get_proxy(self):
url = 'http://api.xdaili.cn/xdaili-api//greatRecharge/getGreatIp?spiderId=c0fc4a1f24e245dc85fd15c355fc3530&orderno=YZ20191237434d6v6Z&returnType=1&count=1'
text = requests.get(url).text
print('请求代理:',"http://"+text.strip())
if '频繁' not in text:
proxy = {
'http':"http://"+text.strip(),
'https':'https://'+text.strip()
}
self.proxy = proxy
elif '10032' in text:
print(text)
print('代理用完')
self.proxy = 'None'
elif '频繁' in text:
print('\n5秒后再获取代理\n')
time.sleep(5.5)
self.get_proxy()
def get_detail(self,jid,lid):
url = 'https://www.zhipin.com/view/job/card.json?jid={jid}&lid={lid}&type=2'.format(jid=jid,lid=lid)
self.agent = random.choice(self.USER_AGENT_LIST)
self.session.headers.update(self.headers)
try:
response = self.session.get(url, proxies=self.proxy, headers=self.headers, timeout=5)
if '请求成功' in response.text:
html = response.json()['html']
return html
elif self.proxy=='None':
return None
else:
self.get_proxy()
self.get_detail(self,jid,lid)
except Exception as e:
print('\n\n遇到错误',e)
self.get_proxy()
self.get_detail(jid,lid)
def getpage(self, factor, page):
self.agent = random.choice(self.USER_AGENT_LIST)
self.session.headers.update(self.headers)
url = 'https://www.zhipin.com/gongsir/{factor}.html?page={page}&ka=page-{page}'.format(factor=factor,page=page)
try:
response = self.session.get(url, proxies=self.proxy, timeout=5, headers=self.headers)
html = response.text
if '公司简介' in html:
print('----------------获取{}列表页{}page\n'.format(factor,page))
return html
elif self.proxy=='None':
return None
elif '过于频繁' in html or '您的账号安全' in html:
print('====过于频繁')
self.get_proxy()
self.getpage(factor, page)
except:
self.getpage(factor, page)
def parse_detail(self, html):
''
html = etree.HTML(html)
text = html.xpath("//div[@class='detail-bottom']//text()")
text = ''.join((s.strip() for s in text))
if len(text)>20:
print('\t\t解析到详情页',text[0:20])
return text
def parse_page(self, html):
html = etree.HTML(html)
lis = html.xpath("//div[@class='job-list']//li")
for li in lis:
title = li.xpath(".//h3[@class='name']//div[@class='job-title']//text()")[0] if len(li.xpath(".//h3[@class='name']//div[@class='job-title']//text()")) else None
salary = li.xpath(".//h3[@class='name']//span[@class='red']//text()")[0] if len(li.xpath(".//h3[@class='name']//span[@class='red']//text()")) else None
location = li.xpath(".//div[@class='info-primary']/p/text()")[0] if len(li.xpath(".//div[@class='info-primary']/p/text()"))>0 else None
require = li.xpath(".//div[@class='info-primary']/p/text()")[1] if len(li.xpath(".//div[@class='info-primary']/p/text()"))>1 else None
edu = li.xpath(".//div[@class='info-primary']/p/text()")[2] if len(li.xpath(".//div[@class='info-primary']/p/text()"))>2 else None
company = html.xpath('//div[@class="info-primary"]/h1//text()')[0] if len(html.xpath('//div[@class="info-primary"]/h1//text()')) else None
jid = li.xpath("./a/@data-jid")[0]
lid = li.xpath("./a/@data-lid")[0]
url = 'https://www.zhipin.com/'+li.xpath("./a/@href")[0] if len(li.xpath("./a/@href")) else None
self.num += 1
print('\t',self.num,'获得:'+title)
detail_html = self.get_detail(lid=lid,jid=jid)
if detail_html:
infor = self.parse_detail(detail_html)
with open('{}boss.txt'.format(company[:2]),'a',encoding='utf-8',errors='replace') as f1:
f1.write('\t'.join(str(_) for _ in (title, salary, company, edu, location, require, infor, url)) + '\n')
item ={
'title': title,
'salary': salary,
'location': location,
'require': require,
'edu': edu,
'company': company,
'infor':infor,
'url': url
}
with open('{}json.txt'.format(company[:2]),'a',errors='replace',encoding='utf-8') as f2:
f2.write(json.dumps(item,indent=4,ensure_ascii=False)+"\n")
else:
pass
def update_company(self):
f = open('company_url.txt','w',encoding='utf-8')
for page in range(1,30):
r = requests.get('https://www.zhipin.com/gongsi/?page={page}&ka=page-{page}'.format(page=page),headers=self.headers,proxies=self.proxy)
html = etree.HTML(r.text)
if '暂时没有' in r.text:
break
elif '过于频繁' in r.text:
self.proxy=self.get_proxy()
self.update_company()
elif '精准匹配' in r.text:
lis = html.xpath("//div[contains(@class,'company-tab-box') and contains(@class,'company-list')]//li")
for li in lis:
com = li.xpath(".//div[@class='conpany-text']/h4/text()")
url = li.xpath(".//div[@class='sub-li']/a[1]/@href")
if com and url:
com=com[0]
url=url[0].replace('/gongsi/','').replace('.html','')
self.company[com]=url
f.write(com+'\t'+url+'\n')
f.close()
def load_company(self):
with open('company_url.txt','r',encoding='utf-8') as f:
companylist = f.readlines()
for i in companylist:
if i:
com = i.split("\t")[0]
url = i.split("\t")[1].strip()
self.company[com]=url
def main(self):
for key in self.company:
if self.proxy=='None':
break
value=self.company[key]
for page in range(1,31):
if self.proxy=='None':
break
html = self.getpage(value,page)
time.sleep(1)
if html:
self.parse_page(html)
if __name__ == '__main__':
boss = BossSpider()
boss.main()```