python3.x爬虫学习：股票数据定向爬虫笔记

import requests
from bs4 import BeautifulSoup
import traceback
import re

def getHTMLtext(url, code="utf-8"):
	try:
		r =requests.get(url)
		r.raise_for_status()
		r.encoding = code
		print("test")
		return r.text
	except:
		return ""


def getStockList(list,stockURL):
	html = getHTMLtext(stockURL,"GB2312")
	print("getstockList start")
	soup = BeautifulSoup(html,'html.parser')
	a = soup.find_all('a')
	for i in a:
		try:
			href = i.attrs['href']
			list.append(re.findall(r"[s][hz]\d{6}",href)[0])
		except:
			continue


def getStockInfo(list,stockURL,filePath):
	count = 0
	for stock in list:
		url = stockURL + stock +".html"
		html =getHTMLtext(url)
		try:
			if html=="":
				continue

			infoDict = {}
			soup = BeautifulSoup(html,"html.parser")
			stockInfo = soup.find('div',attrs={'class':'stock-bets'})

			name = stockInfo.find_all(attrs={'class':'bets-name'})[0]
			infoDict.update({'股票名称': name.next.split()[0]})

			keylist =stockInfo.find_all('dt')
			vauleList = stockInfo.find_all('dd')
			for i in range(len(keylist)):
				key =keylist[i].text
				vaule = vauleList[i].text
				infoDict[key]= vaule

			with open(filePath,'a',encoding='utf-8') as f:
				f.write( str(infoDict) + '\n')
				count= count+1
				print("\r当前进度: {:.2f}%".format(count*100/len(list,end="")))
		except:
			count =count +1
			print("\r当前进度: {:.2f}%".format(count*100/len(list)),end="")
			continue



def main():
	print("start")
	stock_list_url='http://quote.eastmoney.com/stocklist.html'
	stock_info_url = 'https://gupiao.baidu.com/stock/'
	output_file = 'D:/BaiduStockInfo.txt'
	slist=[]
	getStockList(slist,stock_list_url)
	getStockInfo(slist,stock_info_url,output_file)
	print("end")


main()