python入门爬虫-新笔趣阁小说下载

from lxml import etree
import requests

header = {
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Content-Type": "application/x-www-form-urlencoded;charset=UTF-8",
    "Accept-Encoding": "gzip, deflate",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
    "Cache-Control": "no-cache",
    "Connection": "keep-alive",
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36"
}


def trim(text_array):
    return "".join("".join(text_array).split())


class xiaoshuo():

    def getText(self, url, charset="utf-8"):
        return str(requests.get(url, headers=header).content, charset)

    def __init__(self, page):
        r = self.getText(page)

        html = etree.HTML(r)
        self.book_name = trim(html.xpath('//*[@id="info"]/h1/text()'))
        self.book_author = trim(html.xpath('//*[@id="info"]/p[1]/text()'))
        self.book_desc = trim(html.xpath('//*[@id="intro"]/p[position()>1]/text()'))

        a = html.xpath('//*[@id="list"]/dl/dd/a')
        self.chapters = []
        for a1 in a:
            chapter = {'name': a1.xpath('./text()')[0], 'link': 'http://www.xbiquge.la' + a1.xpath('./@href')[0]}
            self.chapters.append(chapter)

    def __writeChapter(self, lines):
        i = 0
        for line in lines:
            lines[i] = "".join(line.split()) + "\n"
            i += 1
        with open('D:\\%s.txt' % self.book_name, 'a', encoding='utf-8') as f:
            f.writelines(lines)

    def __findContent(self, chapter):
        content = etree.HTML(self.getText(chapter['link'])).xpath('//*[@id="content"]/text()')
        content.insert(0, chapter['name'] + '\n')
        self.__writeChapter(content)
        print(chapter['name'] + '下载完成！')

    def down(self):
        for chapter in self.chapters:
            self.__findContent(chapter)


if __name__ == '__main__':
    print("\n\t\t欢迎使用《新笔趣阁》 http://www.xbiquge.la 小说下载小工具\n\n\t\t作者:mbh \t时间:2017-05-06\n")
    print("*************************************************************************")

    # 小说地址
    # target_url = str(input("请输入小说目录下载地址:\n"))

    page_link = 'http://www.xbiquge.la/10/10489/'

    xs = xiaoshuo(page_link)
    xs.down()