在之前的文章中，提到如何爬取雪球网用户的股票数据，但是由于爬取过程中，会存在一些问题，比如由于网站设有反爬虫机制，所以会导致在爬取过程中收到403的response，因此在爬取的时候需要做一些伪装，首先要做的就是建立一个代理ip池，通过代理ip对网站进行访问，从而防止由于单一ip短时间内频繁访问而被网站给ban掉。
但是要获取稳定又多的ip，而且实现每次的ip更新，因此我们要先写一个爬虫，把代理ip网站的ip爬下来，然后封装进我们的ip池里。
这里我选择的是西刺代理，大家也可以自行选择其他的代理ip网站。
西刺代理页面截图
以上是西刺代理的页面截图，页面相对规整，首先进行测试，发现该网站没有采用json的传参方式，但是由于页面十分的规整，因此这里考虑直接对html页面进行爬取，首先先看一下页面的html源码。
页面部分html源码
通过以上我们可以发现，页面相对而言较为规整，每个tr下面就包含了一样数量的td，且，每个tr里的第二个td和第三个td以及倒数第三个td内包含了我们所需要的ip，端口以及ip类型。
接下来就直接上代码了。
#设置ip池#
def get_ip():
url = 'http://www.xicidaili.com/'
headers = {
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36',
'Cookie':'_free_proxy_session=BAh7B0kiD3Nlc3Npb25faWQGOgZFVEkiJTI3MGZjZTNkM2I2NWNhY2M5MWZkYmY0NDhkMGU4MTExBjsAVEkiEF9jc3JmX3Rva2VuBjsARkkiMXhjRldkci9BWGxZTHEzSittMGczMVBiYzhIcVdHTVlKaGFkamRQK1RZVXc9BjsARg%3D%3D--0f372ba6b8e9956e504653078cc02c497684f04e; Hm_lvt_0cf76c77469e965d2957f0553e6ecf59=1539521254,1539521263,1539567080,1539577822; Hm_lpvt_0cf76c77469e965d2957f0553e6ecf59=1539578075'
}
html = requests.get(url,headers=headers)
soup = BeautifulSoup(html.text)
ips = soup.find_all('tr')
result_ls = []
for i in range(2,len(ips)):
ip_info = ips[i]
tds = ip_info.find_all('td')
if len(tds) > 0:
result_ls.append(tds[1].text+':'+tds[2].text)
else:
continue
return result_ls
首先爬取页面上所有的td，将里面的第一个与第二个td取出，并进行拼接，然后再把结果存入到数组中，最后返回result_ls这个数组作为ip池。
def result(user_id,ip):
#根据获取的uid分别爬取用户的股票信息#
url2 ='https://xueqiu.com/v4/stock/portfolio/stocks.json'
user_agent_list=[ "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 "
"(KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1",
"Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 "
"(KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 "
"(KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 "
"(KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6",
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 "
"(KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 "
"(KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5",
"Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 "
"(KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 "
"(KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 "
"(KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 "
"(KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 "
"(KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 "
"(KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 "
"(KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 "
"(KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 "
"(KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 "
"(KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 "
"(KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24",
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 "
"(KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24"]
#设置参数#
params = {
'size':10000,
'type':1,
'pid':-1,
'category':2,
'uid':str(user_id)
}
#请求头文件#
headers2 = {
'User-Agent':choice(user_agent_list),
'Referer': 'https://xueqiu.com/u/'+str(user_id),
'Connection': 'keep-alive',
'cache-control':'no-cache',
'Cookie':cookies
}
#设置代理#
proxies = {'http':ip}
#发送请求#
response2 = requests.get(url2,headers=headers2,params=params,proxies=proxies)
time.sleep(5)
print("响应码："+str(response2.status_code) + '使用ip:'+ip)
stockInfo = []
if response2.status_code == 400:
pass
else:
text2 = json.loads(response2.text)
stockInfo = text2['stocks']
return stockInfo
之后再从ip池中选取一个ip作为参数传入到代理中。
最后上完整代码，有问题或者改进意见欢迎私信共同探讨。
# -*- coding: utf-8 -*-
"""
Created on Sat Oct 13 18:33:35 2018
@author: Chen.Xiao
"""
import requests
import json
import pandas as pd
import numpy as np
import time
import ssl
from random import choice
from bs4 import BeautifulSoup
#防止ssl报错#
ssl._create_default_https_context = ssl._create_unverified_context
cookies = '_ga=GA1.2.1962796314.1538825138; device_id=516f0c95e19bf48ba37ca2f3e68811ed; __utmz=1.1538825152.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); s=fi17cigo1a; bid=5ab5c3e86b4b2c95f32bb04bd50357c3_jn791euz; aliyungf_tc=AQAAAMdnpDWO5QEA/c5CMbWjR/czsdk4; Hm_lvt_1db88642e346389874251b5a1eded6e3=1554460329; __utmc=1; xq_a_token=3450822dc3b6c0b631c3ba4768fcddac23c054d7; xq_a_token.sig=f1ZlcbP6BFkUA32I1cexLDa2KTk; xq_r_token=0de8d3b6155ce156310ff6d4e214d4532198ccec; xq_r_token.sig=8xgxBORSc2oawdjl5r3ksadcr9s; _gid=GA1.2.684281849.1555120269; u=231555120270276; _gat=1; __utma=1.1962796314.1538825138.1554640094.1555121068.23; __utmt=1; __utmb=1.2.10.1555121068; Hm_lpvt_1db88642e346389874251b5a1eded6e3=1555121080'
#设置ip池#
def get_ip():
url = 'http://www.xicidaili.com/'
headers = {
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36',
'Cookie':'_free_proxy_session=BAh7B0kiD3Nlc3Npb25faWQGOgZFVEkiJTI3MGZjZTNkM2I2NWNhY2M5MWZkYmY0NDhkMGU4MTExBjsAVEkiEF9jc3JmX3Rva2VuBjsARkkiMXhjRldkci9BWGxZTHEzSittMGczMVBiYzhIcVdHTVlKaGFkamRQK1RZVXc9BjsARg%3D%3D--0f372ba6b8e9956e504653078cc02c497684f04e; Hm_lvt_0cf76c77469e965d2957f0553e6ecf59=1539521254,1539521263,1539567080,1539577822; Hm_lpvt_0cf76c77469e965d2957f0553e6ecf59=1539578075'
}
html = requests.get(url,headers=headers)
soup = BeautifulSoup(html.text)
ips = soup.find_all('tr')
result_ls = []
for i in range(2,len(ips)):
ip_info = ips[i]
tds = ip_info.find_all('td')
if len(tds) > 0:
result_ls.append(tds[1].text+':'+tds[2].text)
else:
continue
return result_ls
ips = get_ip()
print(ips)
#存储股票id#
stocks = {
'code':[],
'stockName':[],
'exchange':[]
}
def get_uid():
#设置url_1获取投资者的id#
url = 'https://xueqiu.com/recommend/user/industry.json?id=47&_=1555121107441'
#设置请求头文件#
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36',
'Cookie': cookies,
'Referer': 'https://xueqiu.com/people'
}
#获取userid#
user_id=[]
#发送请求#
print('开始发送请求...')
response = requests.get(url,headers=headers)
print("响应码："+str(response.status_code))
if response.status_code == 400:
pass
#print(response.text)
else:
text = json.loads(response.text)
users = text['industries'][0]['users']
for i in range(len(users)):
user_id.append(users[i]['id'])
print(user_id)
return user_id
def result(user_id,ip):
#根据获取的uid分别爬取用户的股票信息#
url2 ='https://xueqiu.com/v4/stock/portfolio/stocks.json'
user_agent_list=[ "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 "
"(KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1",
"Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 "
"(KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 "
"(KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 "
"(KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6",
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 "
"(KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 "
"(KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5",
"Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 "
"(KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 "
"(KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 "
"(KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 "
"(KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 "
"(KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 "
"(KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 "
"(KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 "
"(KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 "
"(KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 "
"(KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 "
"(KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24",
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 "
"(KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24"]
#设置参数#
params = {
'size':10000,
'type':1,
'pid':-1,
'category':2,
'uid':str(user_id)
}
#请求头文件#
headers2 = {
'User-Agent':choice(user_agent_list),
'Referer': 'https://xueqiu.com/u/'+str(user_id),
'Connection': 'keep-alive',
'cache-control':'no-cache',
'Cookie':cookies
}
#设置代理#
proxies = {'http':ip}
#发送请求#
response2 = requests.get(url2,headers=headers2,params=params,proxies=proxies)
time.sleep(5)
print("响应码："+str(response2.status_code) + '使用ip:'+ip)
stockInfo = []
if response2.status_code == 400:
pass
else:
text2 = json.loads(response2.text)
stockInfo = text2['stocks']
return stockInfo
def main():
uid = get_uid()
for i in range(len(uid)):
print('采集第'+str(i)+'个数据')
ip = choice(ips)
user_stockInfo = result(uid[i],ip)
for j in range(len(user_stockInfo)):
stocks['code'].append(user_stockInfo[j]['code'])
stocks['stockName'].append(user_stockInfo[j]['stockName'])
stocks['exchange'].append(user_stockInfo[j]['exchange'])
stocks_df = pd.DataFrame(stocks)
stocks_df.to_csv('./stocksInfo.csv')
if __name__ =="__main__":
#main()
uid = get_uid()
for i in range(len(uid)):
print('采集第'+str(i)+'个数据')
ip = choice(ips)
user_stockInfo = result(uid[i],ip)a
for j in range(len(user_stockInfo)):
stocks['code'].append(user_stockInfo[j]['code'])
stocks['stockName'].append(user_stockInfo[j]['stockName'])
stocks['exchange'].append(user_stockInfo[j]['exchange'])
stocks_df = pd.DataFrame(stocks)
stocks_df.to_csv('./stocksInfo.csv')