今天按照视频上的步骤爬起股票数据时,控制台出现了
name = stockInfo.find_all(attrs={'class':'bets-name'})[0]
AttributeError: 'NoneType' object has no attribute 'find_all'
的错误信息。
百度搜索了一下,发现大家都有这样的错误提示。在网上暂时没有找到解决这个错误的办法。
所以我就把自己的想法和大家分享一下,希望对大家有所帮助,同时也希望大家能提出意见和建议。
import requests
from bs4 import BeautifulSoup
import re
import traceback
def getHTMLText(url,code='utf-8'):
try:
r = requests.get(url)
r.raise_for_status()
r.encoding = code
return r.text
except:
return ""
def getStockList(lst,stockURL):
html = getHTMLText(stockURL,'GB2312')
soup = BeautifulSoup(html,'html.parser')
a = soup.find_all('a')
for i in a:
try:
href = i.attrs['href']
lst.append(re.findall(r"[s][hz]\d{6}",href)[0])
except:
continue
def getStockInfo(lst,stockURL,fpath):
count = 0
for stock in lst:
url = stockURL + stock + ".html"
html = getHTMLText(url)
try:
if html == "":
continue
infoDict = {}
soup = BeautifulSoup(html,'html.parser')
stockInfo = soup.find('div',attrs={'class':'stock-bets'})
name = stockInfo.find_all(attrs={'class':'bets-name'})[0]
infoDict.update({'股票名称': name.text.split()[0]})
keyList = stockInfo.find_all('dt')
valueList = stockInfo.find_all('dd')
for i in range(len(keyList)):
key = keyList[i].text
val = valueList[i].text
infoDict[key] = val
with open(fpath,'a',encoding='utf-8') as f:
f.write(str(infoDict) + '\n')
count = count + 1
print("\r当前进度:{:.2f}%".format(count * 100 / len(lst)),end="")
except:
count = count + 1
print("\r当前进度:{:.2f}%".format(count * 100 / len(lst)), end="")
traceback.print_exc()
continue
def main():
stock_list_url = 'http://quote.eastmoney.com/stocklist.html'
stock_info_url = 'https://gupiao.baidu.com/stock/'
output_file = 'D://BaiduStockInfo.txt'
slist = []
getStockList(slist,stock_list_url)
getStockInfo(slist,stock_info_url,output_file)
main()
错误提示分析:属性错误,'NoneType' 对象没有属性 'find_all' 。这种类型的错误在以前的博客中已经和大家介绍过两个,同样现在这个错误提示依然和以前一样。
错误提示出现的原因:因为上一行代码:stockInfo = soup.find('div', attrs={'class' : 'stock-bets'}) 中获取的数据类型中有空类型'NoneType’ ,通过 print(type(stockInfo)) 可以输出变量stockInfo的类型,发现输出了
既然知道了原因,那么就可以针对原因提出解决方法。
我的办法是过滤掉
在stockInfo = soup.find('div', attrs={'class' : 'stock-bets'})代码下面增加一个if 判断, 用isinstance()函数将空类型过滤掉。
即:
if isinstance(stockInfo,bs4.element.Tag):
然后缩进后续相关的代码,注意在使用isinstance()时,参数2:bs4.element.Tag是参数1:stockInfo要匹配的类型。在使用参数2时要在开头引入bs4模块,即import bs4
import requests from bs4 import BeautifulSoup import re import traceback import bs4 #注意点1:引入模块 def getHTMLText(url,code='utf-8'): try: r = requests.get(url) r.raise_for_status() r.encoding = code return r.text except: return "" def getStockList(lst,stockURL): html = getHTMLText(stockURL,'GB2312') soup = BeautifulSoup(html,'html.parser') a = soup.find_all('a') for i in a: try: href = i.attrs['href'] lst.append(re.findall(r"[s][hz]\d{6}",href)[0]) except: continue def getStockInfo(lst,stockURL,fpath): count = 0 for stock in lst: url = stockURL + stock + ".html" html = getHTMLText(url) try: if html == "": continue infoDict = {} soup = BeautifulSoup(html,'html.parser') stockInfo = soup.find('div',attrs={'class':'stock-bets'}) if isinstance(stockInfo,bs4.element.Tag): # 注意点2:增加一个if判断语句以及后续代码的缩进 name = stockInfo.find_all(attrs={'class':'bets-name'})[0] infoDict.update({'股票名称': name.text.split()[0]}) keyList = stockInfo.find_all('dt') valueList = stockInfo.find_all('dd') for i in range(len(keyList)): key = keyList[i].text val = valueList[i].text infoDict[key] = val with open(fpath,'a',encoding='utf-8') as f: f.write(str(infoDict) + '\n') count = count + 1 print("\r当前进度:{:.2f}%".format(count * 100 / len(lst)),end="") except: count = count + 1 print("\r当前进度:{:.2f}%".format(count * 100 / len(lst)), end="") traceback.print_exc() continue def main(): stock_list_url = 'http://quote.eastmoney.com/stocklist.html' stock_info_url = 'https://gupiao.baidu.com/stock/' output_file = 'D://BaiduStockInfo.txt' slist = [] getStockList(slist,stock_list_url) getStockInfo(slist,stock_info_url,output_file) main()