PYTHON股票定向爬虫（2020年后针对廖雪峰老师的爬虫教程练习）

最近跟着廖雪峰老师的python爬虫视频学习爬虫，发现很多之前的网站由于经过了改写而导致现在爬虫时难以获取页面。因此特别写了这些笔记，提供给2020年之后学习爬虫的小伙伴看～
老师在课程中提到的东方财富网，网址已被重写，现在应该是这个：
http://quote.eastmoney.com/stock_list.html
百度股票现在打开都会反错404，因此采用老虎股票带去，网址如下：
https://www.laohu8.com/stock/
下面完整代码如下：
（记得自己填入自己的文件路径嗷）

import re
from bs4 import BeautifulSoup
import requests
import traceback
import bs4


def getHTMLText(url,code='utf-8'):
    try:
        r = requests.get(url, timeout=30)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        return r.text
    except:
        return ""


def getStockList(lst, stockURL):
    html = getHTMLText(stockURL,'GB2312')
    soup = BeautifulSoup(html, 'html.parser')
    a = soup.find_all('a')
    for i in a:
        try:
            href = i.attrs['href']
            lst.append(re.findall(r"[s][hz]\d{6}", href)[0])
        except:
            continue


def getStockInfo(lst, stockURL, fpath):
    count = 0
    for i in range(len(lst)):
        code = re.search(r'\d{6}',lst[i])
        stock_info_url_final = stockURL + code.group(0)
        try:
            if stock_info_url_final == "":
                continue
            html=getHTMLText(stock_info_url_final)
            infoDict = {}
            soup = BeautifulSoup(html, 'html.parser')
            stockInfo = soup.find('div', attrs={'class': 'detail-data'})
            name = soup.find('h1', attrs={'class': 'name'})
            price=soup.find('span',attrs={'class':'latest'})
            ratio=soup.find('span',attrs={'class':'change'})
            if ratio!=0:
                testDict={}
                testDict[name.text]=price.text
            if isinstance(stockInfo, bs4.element.Tag):
                keyList = stockInfo.find_all('dt')
                valueList = stockInfo.find_all('dd')
                for i in range(len(keyList)):
                    key = keyList[i].text
                    val = valueList[i].text
                    infoDict[key] = val

                with open(fpath, "a", encoding='utf-8')as f:
                    f.write(str(testDict)+'\n'+str(infoDict) + '\n')
                    count = count + 1
                    print("\r当前进度:{:.2f}%".format(count * 100 / len(lst)), end="")
        except:
            traceback.print_exc()
            continue


def main():
    stock_list_url = 'http://quote.eastmoney.com/stock_list.html'
    stock_info_url = 'https://www.laohu8.com/stock/'
    output_file = '/Users/oltremare/Desktop/股票信息/股票.txt'
    slist = []
    getStockList(slist, stock_list_url)
    getStockInfo(slist, stock_info_url, output_file)


main()