Python真香之爬取古诗文网

最近在学习Python相关，学习了基本的语法后想搞点事情试试，所以来爬取下古诗文网中的相关作者信息
准备资料：
爬取目标：爬取古诗文网的唐代作者的信息
目标分析：

作者列表页面

一级页面是所有唐代作者的列表，点击名字后会跳转到作者详情页，所以这一页我们要做的就是解析出每一项作者详情页url，并且自动翻页

作者详情页

上述是作者详情页，其中主要包含作者名字，作者简述和作者生平故事，这一页我要做的事情是解析这些数据并保存到本地文件；
详细代码

#!/usr/bin/env python2.7
import requests
import codecs
import json
from bs4 import BeautifulSoup

DOWNLOAD_URL = 'https://so.gushiwen.org/authors/Default.aspx?p=1&c=%E5%94%90%E4%BB%A3'

def download_page(url):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.80 Safari/537.36'
    }
    data = requests.get(url, headers=headers).content
    return data
def parseHtml(html):
    soup = BeautifulSoup(html, features="html.parser")
    nextPage = 'https://so.gushiwen.org' + soup.find('a', attrs={'class': 'amore'}).get('href')
    leftDiv = soup.find('div', attrs={'class': 'main3'}).find('div', attrs={'class': 'left'})
    data = []
    for item in leftDiv.find_all('div', attrs={'class': 'sonspic'}):
        data.append('https://so.gushiwen.org' + item.find('a')['href'])
    return data, nextPage
def buildJson(keys, values):
    dictionary = dict(zip(keys, values))
    return json.dumps(dictionary)

def parseAuthorHtml(html):
    soup = BeautifulSoup(html, features="html.parser")
    all = soup.find('div', attrs={'class': 'main3'}).find('div', attrs={'class': 'left'})
    author = all.find('div', attrs={'class': 'sonspic'}).find('div', attrs={'class': 'cont'}).find('b').getText()
    desc = all.find('div', attrs={'class': 'sonspic'}).find('div', attrs={'class': 'cont'}).find('p').getText()
    sons = all.find('div',attrs={'class': 'sons'})
    yishiUrl = None
    if sons:
        yishiUrl = sons.get('id')
        if yishiUrl:
            yishiUrl = 'https://so.gushiwen.org/authors/ajaxziliao.aspx?id=' + all.find('div',attrs={'class': 'sons'})['id'].replace('fanyi','')
        else:
            yishiUrl = None
    return author, desc, yishiUrl

def parseAuthorMore(html):
    soup = BeautifulSoup(html, features='html.parser')
    yishi = []
    if soup.find('div', attrs={'clase', 'contyishang'}):
        for p in soup.find('div', attrs={'clase', 'contyishang'}).find_all('p'):
            yishi.append(p.getText())
    return ''.join(yishi)

def main():
    url = DOWNLOAD_URL
    keys = ['author', 'desc', 'story']
    with codecs.open('author.json', 'w', encoding='utf-8') as fp:
        fp.write('[')
        while url:
            html = download_page(url)
            data, url = parseHtml(html)
            for item in data:
                authorHtml = download_page(item)
                author, desc, yishiUrl = parseAuthorHtml(authorHtml)
                if yishiUrl:
                    yishiHtml = download_page(yishiUrl)
                    yishi = parseAuthorMore(yishiHtml)
                else:
                    yishi = ""
                fp.write(buildJson(keys, [author, desc,yishi]))
                fp.write(',')
        fp.write(']')

if __name__ == '__main__':
    main()

整体思路：请求-->解析数据-->存储
请求使用了requests库，佯装正常的浏览器请求，获取到数据之后，将数据交给BeautifulSoup来做数据解析，主要是找到html中的相关标签并获取其相关属性值和标签内容，然后组装成自己要的目标作者的详情页url，之后访问详情页，再获取数据并存储再本地