本文通过抓取常见的User-Agent(用户代理),将其写入列表并保存为json格式文件,且将代码进行了封装,方便以后抓取数据时动态的更新请求头中的User-Agent,模拟真实的浏览器发送请求,从一方面避免抓取数据时反爬的干扰。
# *********************** User-Agent 爬取 ********************************
import requests
from bs4 import BeautifulSoup
import re
import json
import urllib.request
class UserAgentSpider(object):
def __init__(self):
self.headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36"}
self.url = "https://blog.csdn.net/jamesaonier/article/details/89003053"
self.user_agent_list = []
self.validation_user_agent_list = []
# 1、发送网络请求 获取数据
def data_request(self):
return requests.get(self.url,headers = self.headers).content.decode("utf-8")
# 2、数据解析
def data_parse(self,data):
parse_data = BeautifulSoup(data,"lxml")
all_agent = parse_data.select("div p")[1:12]
for agent in all_agent:
pattern = re.compile("\n")
each_agent = pattern.split(agent.get_text())[1:]
for each in each_agent:
self.user_agent_list.append(each)
# 3、检验user-agent的可用性
def test_validation(self):
for agent in self.user_agent_list:
try:
url = "https://www.baidu.com/"
headers = {}
headers["User-Agent"] = agent
requests.get(url,headers = headers)
self.validation_user_agent_list.append(agent)
except urllib.request.HTTPError as error:
print(error.code)
# 4、数据存储
def data_save(self):
with open("./User-Agent.json","w",encoding = "utf-8") as fp:
json.dump(self.validation_user_agent_list,fp)
# 4、统筹运行
def run(self):
# 1、请求数据
data = self.data_request()
# 2、数据解析
self.data_parse(data)
# print(self.user_agent_list)
# 3、检验user-agent的可用性
self.test_validation()
# 4、数据存储
self.data_save()
if __name__ == "__main__":
UserAgentSpider().run()
代码运行结果:
["Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36 OPR/26.0.1656.60", "Opera/8.0 (Windows NT 5.1; U; en)", "Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/2.0.0 Opera 9.50", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 9.50", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0", "Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11", "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36", "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.11 TaoBrowser/2.0 Safari/536.11", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E; LBBROWSER)\"", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)", "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0", "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; SE 2.X MetaSr 1.0)", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Maxthon/4.4.3.4000 Chrome/30.0.1599.101 Safari/537.36", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 UBrowser/4.0.3214.0 Safari/537.36"]
by CyrusMay 2020 04 24
从前只想装懂
装作什么都懂
懂得生存的规则之后
却只想要 都不懂
——————五月天——————