import requests
from bs4 import BeautifulSoup
def get_chapters():
"""
获取小说章节链接
:return:
"""
root_url = "http://www.89wx.cc/17/17277/" # 小说网站根目录
r = requests.get(root_url)
r.encoding = 'gbk' # 查看小说网站的编码，为 gbk
soup = BeautifulSoup(r.text, 'html.parser')
links = []
# 查看网页，得知小说章节都是放在 dd 标签中的 a 标签
for dd in soup.find_all("dd"):
link = dd.find("a")
if not link:
continue
links.append(('http://www.89wx.cc' + link["href"], link.get_text()))
return links
def get_chapter_content(url):
"""
获取小说章节内容
:param url:
:return:
"""
r = requests.get(url)
r.encoding = 'gbk'
soup = BeautifulSoup(r.text, "html.parser")
text = soup.find("div", id="content").get_text()
return text
novel_chapters = get_chapters()
for chapter in novel_chapters:
url, title = chapter
with open(title + ".txt", "w", encoding="utf-8") as f:
f.write(get_chapter_content(url))
# break
