Python 爬虫实战 —— 爬取小说

import requests
from bs4 import BeautifulSoup


def get_chapters():
    """
    获取小说章节链接
    :return:
    """
    root_url = "http://www.89wx.cc/17/17277/"  # 小说网站根目录
    r = requests.get(root_url)
    r.encoding = 'gbk'  # 查看小说网站的编码，为 gbk
    soup = BeautifulSoup(r.text, 'html.parser')

    links = []
    # 查看网页，得知小说章节都是放在 dd 标签中的 a 标签
    for dd in soup.find_all("dd"):
        link = dd.find("a")
        if not link:
            continue
        links.append(('http://www.89wx.cc' + link["href"], link.get_text()))
    return links


def get_chapter_content(url):
    """
    获取小说章节内容
    :param url:
    :return:
    """
    r = requests.get(url)
    r.encoding = 'gbk'
    soup = BeautifulSoup(r.text, "html.parser")
    text = soup.find("div", id="content").get_text()
    return text


novel_chapters = get_chapters()
for chapter in novel_chapters:
    url, title = chapter
    with open(title + ".txt", "w", encoding="utf-8") as f:
        f.write(get_chapter_content(url))
    # break