每周分享【一】 用Python爬取天津大学课程表

需要的库

import time
import bs4
import xlwt  #用来写入Excel
from selenium import webdriver  #模拟浏览器行为
from bs4 import BeautifulSoup
import getpass  #隐藏输入的密码

# 还要单独安装chromedriver

思路：

由于校园办公网的课程表页是用js渲染出来的而不是静态页面，所以不直接用requests库去get页面，那样得到的网页源代码是不经js渲染的，而校园网又没有提供接口，所以只能暴力解决。Selenium神器登场

• 定位登录页面元素
• 自动登录
• 自动进入课程表页
• 获取渲染后的网页源代码
• BeautifulSoup解析
• 写入数据

上代码

import time
import bs4
import xlwt
from selenium import webdriver
from bs4 import BeautifulSoup
import getpass

#获取渲染后的代码
browser = webdriver.Chrome()

url = "http://classes.tju.edu.cn"
username = input("学号：")
password = getpass.getpass("密码：")

browser.get(url)
time.sleep(2)
browser.find_element_by_name('username').send_keys(username)
time.sleep(2)
browser.find_element_by_name('password').send_keys(password)
time.sleep(2)
browser.find_element_by_name('submit').click()
time.sleep(3)
browser.switch_to.frame('iframeMain')
browser.find_element_by_xpath("//div[@class='inner-list-2']/a[@href='/eams/courseTableForStd.action']").click()
time.sleep(3)
html = browser.page_source

#定位课程表
soup = BeautifulSoup(html, 'lxml')
# print(soup)
table = soup.find('table', attrs={'id': 'manualArrangeCourseTable'})
trs = table.find_all('tr')[1:]
# tds = trs[2].find_all('td')
# print(tds[1].text)

# 读入课程表
data = {
    '0': [],
    '1': [],
    '2': [],
    '3': [],
    '4': [],
    '5': [],
}

for i in range(0, 12):
    if i % 2 != 0:
        continue
    if isinstance(trs[i], bs4.element.Tag):
        tds = trs[i].find_all('td')
        for j in range(1, 8):
            a = str(int(i/2))
            # print(a)
            data[a].append(tds[j].text)
# print(data)
browser.close()
# 初始化表格
workbook = xlwt.Workbook(encoding='utf-8')
worksheet = workbook.add_sheet('课程表')
worksheet.col(0).width = 4000
for i in range(1, 6):
    worksheet.col(i).width = 8000

style = xlwt.XFStyle()
font = xlwt.Font()
font.name = "宋体"
font.height = 13 * 20
alignment = xlwt.Alignment()
alignment.horz = xlwt.Alignment.HORZ_CENTER
alignment.vert = xlwt.Alignment.VERT_CENTER
style.font = font
style.alignment = alignment

worksheet.write(0, 0, "节次", style)
for i in range(1, 8):
    a = str(i)
    worksheet.write(0, i, "星期" + a, style)

for j in range(1, 6):
    b = str(j)
    c = str(2 * j)
    worksheet.write(j, 0, "第" + b + "节" + '\n' + "第" + c + "节", style)

for i in range(1, 6):
    for j in range(1, 8):
        worksheet.write(i, j, data[str(i-1)][j-1], style)

workbook.save('课程表.xls')

终端运行！！！！！！