import requests
from bs4 import BeautifulSoup
import re
import json
import numpy as np
import time
import tushare as ts
import pandas as pd
df1 = ts.get_hs300s()
ss=df1.head(10)
code = ss.code
data = {}
for stkCode in code:
url = "http://stockpage.10jqka.com.cn/" + stkCode[:6] + "/worth/#forecast"
headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"}
html = requests.get(url, headers = headers).text
soup = BeautifulSoup(html, "lxml")
s = soup.find_all(class_="tip clearfix")[0]
pattern = re.compile("预测\d+.*?(\d+.\d+).*?(\d+.\d+%)")
result = re.findall(pattern, s.text)
data[stkCode] = result
t = np.random.randint(10)
time.sleep(t+5)
with open('data.json', 'w+') as f:
json.dump(data, f)
f.close()
with open("data.json", 'r') as f:
d = json.load(f)
f.close()
new_dict = {}
for key, values in d.items():
if len(values) == 0:
values = [[None, None], [None, None]]
elif len(values) == 1:
values.append([None, None])
t = (values[0][0], values[0][1], values[1][0], values[1][1])
new_dict[key] = t
index = ['预测2018年每股收益/元', '每股收益较去年同比增长', '预测2018年净利润/亿元', '净利润较去年同比增长']
df = pd.DataFrame(index=index, data=new_dict)
df = df.T
print(df)
抓取结果如下图