用到的库：requests
爬虫思路
(xxxxxxxxx代表教务网网址，不便传播)
- 创建一个Requests会话对象(所有的爬虫步骤都在该会话下进行)。
s = requests.Session()
- get教务网的登陆页面，返回的头部信息中的cookies信息。
r = s.get(r'http://kdjw.hnust.cn/kdjw/', headers=kdjw_hed)
cookies = r.headers['Set-Cookie'][0:-12]
- 根据获取到cookies信息实时获取当前登录页面的验证码图片信息，并把它的二进制文件信息存储为code.jpg。
r_verifycode = s.get(r'http://xxxxxxxxx/verifycode.servlet', headers=verifycode_hed)
with open('code.jpg','wb') as f:
f.write(r_verifycode.content)
- 请求用户输入验证码，然后把所收到的验证码字符串以及用户输入的学号，密码POST到真正的登录的页面，进而实现模拟登陆。
r_login = s.post(r'http://xxxxxxxxx/Logon.do?method=logon', headers=post_hed, data=login_data)
- 模拟登陆后，直接携带cookies POST发送请求给教务网的成绩信息存储页面,进而获得全部成绩信息的原始数据。
r_grades = s.post(r'http://xxxxxxxxx/xszqcjglAction.do?method=queryxscj', headers=grades_hed, data=form_data)
return r_grades.text
获取到成绩信息的原始数据之后，我们就可以进行数据的清洗和整理了~
下面给出该思路的源代码
（大神勿喷，代码并不能直接运行）
import sys
import io
import requests
from PIL import Image
#改变标准输出的默认编码
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf8')
def get_cookies():
kdjw_hed = {'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
'Accept-Encoding': 'gzip, deflate',
'Accept-Language': 'zh-CN,zh;q=0.9',
'Cache-Control': 'max-age=0',
'Connection': 'keep-alive',
'Host': 'xxxxxxxxx',
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36',
}
r = s.get(r'http://xxxxxxxxx/',headers=kdjw_hed)
#print(r.headers)
#获取到cookies
cookies = r.headers['Set-Cookie'][0:-12]
return cookies
def get_verifycode(cookies):
'根据所得cookies获取验证码'
verifycode_hed = {'Accept': 'image/webp,image/apng,image/*,*/*;q=0.8',
'Accept-Encoding': 'gzip, deflate',
'Accept-Language': 'zh-CN,zh;q=0.9',
'Connection': 'keep-alive',
'Cookie': cookies,
'Host': 'xxxxxxxxx',
'Referer': 'http://xxxxxxxxx/',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36',
}
#验证码获取
r_verifycode = s.get(r'http://xxxxxxxxx/verifycode.servlet', headers=verifycode_hed)
with open('code.jpg','wb') as f:
f.write(r_verifycode.content)
return True
def show_verifycode():
'通过Image模块展示验证码图片，并提示用户输入验证码'
im = Image.open('code.jpg')
im.show()
verifycode = input('请输入验证码: ')
return verifycode
def virtual_login(verifycode, cookies, usr, pwd):
#模拟登陆
login_data = {'useDogCode': '',
'USERNAME': usr,
'PASSWORD': pwd,
'RANDOMCODE': verifycode}
post_hed = {'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
'Accept-Encoding': 'gzip, deflate',
'Accept-Language': 'zh-CN,zh;q=0.9',
'Cache-Control': 'max-age=0',
'Connection': 'keep-alive',
'Content-Length': '86',
'Content-Type': 'application/x-www-form-urlencoded',
'Cookie': cookies,
'Host': 'xxxxxxxxx',
'Origin': 'http://xxxxxxxxx',
'Referer': 'http://xxxxxxxxx/',
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36',
}
r_login = s.post(r'http://xxxxxxxxx/Logon.do?method=logon', headers=post_hed, data=login_data)
r_login.raise_for_status()
return usr
def search_and_get_grades_html(cookies, kksj=''):
'模拟登陆之后，打开成绩查询页面获取成绩数据'
form_data = {'kksj': kksj,
'kcxz': '',
'kcmc': '',
'xsfs': 'qbcj', #全部成绩
'ok': '',
}
grades_hed = {'Accept':
'Cookie': cookies,
'Referer': 'http://xxxxxxxxx/jiaowu/cjgl/xszq/query_xscj.jsp',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36',
}
#成绩查询
r_grades = s.post(r'http://xxxxxxxxx/xszqcjglAction.do?method=queryxscj', headers=grades_hed, data=form_data)
hed_dict = dict(r_grades.headers)
try:
if hed_dict['Content-Length']:
print('学号、密码或验证码输入错误，请重新输入...')
except KeyError:
print('成功获取成绩数据！！！')
return r_grades.text
def get_code_in_file():
global s, cookies_
s = requests.Session()
#cookies
cookies_ = get_cookies()
#更新验证码code.jpg
get_verifycode(cookies_)
return cookies_
def main(verifycode_, cookies_, usr_='', pwd_='', kksj_=''):
#登陆获取数据
if usr_ == '' or pwd_ == '':
username = virtual_login(verifycode_, cookies_)
else:
username = virtual_login(verifycode_, cookies_, usr_, pwd_)
if kksj_ == '':
orignal_html = search_and_get_grades_html(cookies_)
else:
orignal_html = search_and_get_grades_html(cookies_, kksj_)
return orignal_html, username
if __name__ == '__main__':
cookies = get_code_in_file()
show_verifycode()
main(verifycode_=input('验证码：'), cookies_=cookies, usr_=input('学号：'), pwd_=input('密码：'), kksj_='')
公众号：胜言
个人博客：wangsheng.tech