python爬取txt电子书

直接上上代码了，比较简单：

#/usr/bin/evn python3
#-*- coding:utf-8 -*-

import requests
from bs4 import BeautifulSoup

base_url = 'http://www.xxbiquge.com'

user_agent = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36\
    (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36'
header = {'User_Agent':user_agent}
Title = ''

def get_sub_url():
    items = []

    r = requests.get('http://www.xxbiquge.com/32_32006/', headers = header)
    r.encoding = 'utf-8'
    soup = BeautifulSoup(r.text, 'html.parser')

    global Title
    title = soup.find('dt')
    if title != None:
        print(title.string)
        Title = title.string

    for sutitle in soup.find_all('dd'):
        find_dd = sutitle.find('a')
        suburls = {}
        suburls['link'] = find_dd.get('href')
        suburls['title'] = find_dd.string
        items.append(suburls)
    return items

def get_content(url, title):
    r = requests.get(url, headers = header)
    r.encoding = 'utf-8'
    soup = BeautifulSoup(r.text, 'html.parser')
    contents = soup.find_all(id = 'content')
    for content in contents:
        #print(str(content.get_text()).replace('
','\n'))
        with open(Title, "a+") as f:
            f.write(title)
            f.write(str(content.get_text()).replace('
', '\n'))


if __name__ == '__main__':
    sub_urls = get_sub_url()
    for i in sub_urls:
        print("Real link:{}".format(base_url + i['link']), i['title'])
        get_content("{}".format(base_url + i['link']), i['title'])