Python通过selniumwire抓取公开信息全过程
Python抓取数据解析有两种模式，一种是网站前后端融合的情况，利用Beautiful Soup 来解析，即网页显示有什么就能抓取什么，这种方法缺陷是解析速度慢，以及网页结构可能变化要随时修正脚本。另一种是针对前后端分离的情况，网站后端通过接口返回数据给前端解析显示，这种时候我们只需要抓到这个接口，再发起请求就可以得到数据字段比页面显示更丰富、格式更标准的数据。
对前后端分离的网站，在发起请求时会调用自身的接口返回数据给前端，如果是普通网站这时候我们已经可以通过浏览器F12观察接口的header，在脚本里构造好带cookies、header、请求参数params的get/post请求。这里使用高级查询功能调用接口
但在脚本里直接用request请求接口时候会发现，接口只可用一次，每次只返回20条数据。这是因为请求里含分页信息，每页只返回20条，获取更多则需要在请求参数里填写请求第二页第三页。而我们脚本利用构造的请求时，只会返回当页请求，无法返回第二页第三页，仔细观察，原来是请求头headers中构造了一个企查查的自定义字段，每次发起新请求时(包括请求其他页数)该字段都会变化，所以脚本无法直接利用request请求所有数据。
但是我们在浏览器里访问的时候，是可以手动点击下一页下一页的，所以我们可以采用selnium来模拟浏览器的操作，来获得每次请求时网站生成的header中随机自定义字段，再利用request请求调用企查查原生接口获得返回数据。selnium是用脚本模拟人操作浏览器的工具，可以做到和人工操作一样，来完成一些纯请求脚本不好做的事情，支持多种浏览器，这里下载一个谷歌浏览器，再下载对应谷歌浏览器版本的selnium驱动放到Python文件夹里即可。我们在这里除了要模拟浏览器操作，还需要获取请求头headers的信息，所以我们要用的是selniumwire，可以理解为selnium加强版。
技术路径准备好后，考虑抓取流程，即使在浏览器里查看，每次最多也只会返回5000条数据，所以要把这500多W条抓完，就必须把数据分段，这里可以使用企业注册日期来分段，使得每个日期区间的企业数量在5000条内即可，然后脚本中模拟人工查询、翻页操作，再调用request请求解析最后写入数据库。
首先我们要做日期分段功能，把日期切分到每段区间不超过5000条。人工观察了下，越早注册的企业越少，后期成都注册的企业一天就几千条了，所以我按照时间来粗分。2003年以前，以30天为分段；2003-2015以5天为分段；2015以后，每天为分段。期间如果多天分段的大于5000条则进一步拆分为每天分段，如果每天也大于5000则记录到日志中跳过，如果某段结果小于1000条，则扩展该段日期。
导入和定义一些基础信息
import sys
from ast import keyword
from asyncio.windows_events import NULL
from cgitb import small
from cmath import e
from ctypes import addressof
from email import header
import email
import errno
import json
from nturl2path import url2pathname
from os import stat
from pickletools import long1
from re import S
from tkinter import E
from urllib.error import HTTPError
import winreg #windows相关库
#加载自动化测试模块
#from selenium import webdriver
from bs4 import BeautifulSoup #网页解析库
import urllib.request #请求库
import requests
#from lxml import etree
import time
import pymysql
#加载自动化测试模块
from seleniumwire import webdriver
from selenium.webdriver.common.action_chains import ActionChains
import gzip
import datetime
from selenium.webdriver.common.keys import Keys
def get_desktop(): #获得windows桌面路径
key = winreg.OpenKey(winreg.HKEY_CURRENT_USER,r'Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders')
return winreg.QueryValueEx(key, "Desktop")[0]
log_path = get_desktop()+"\\脚本\\log\\log_抓企查查通过搜索抓基础信息.txt"
Paging_path = get_desktop()+"\\脚本\\config\\config_抓企查查通过搜索抓基础信息_分页.txt"
Paging_log_path = get_desktop()+"\\脚本\\log\\log_抓企查查通过搜索抓基础信息_分页_单日大于5000条的日期.txt"
config_path = get_desktop()+"\\脚本\\config\\config_抓企查查通过搜索抓基础信息.ini"
lastDate = datetime.datetime.now()
lastDate = lastDate+datetime.timedelta(days=-15)定义一些selium操作浏览器时选择、点击相关元素的基础方法
# 利用异常捕获和递归实现网络延迟下的稳定抓取
def find_element_by_css_selector(driver,id):
try:
get = driver.find_element_by_css_selector(id)
except:
time.sleep(0.2)
get = find_element_by_css_selector(driver,id)
return get
# 利用异常捕获和递归实现网络延迟下的稳定抓取
def find_element_by_class_name(driver,id):
try:
get = driver.find_element_by_class_name(id)
except:
time.sleep(0.2)
get = find_element_by_class_name(driver,id)
return get
# 利用异常捕获和递归实现网络延迟下的稳定抓取
def find_element_by_link_text(driver,id):
try:
get = driver.find_element_by_link_text(id)
except:
time.sleep(0.2)
get = find_element_by_link_text(driver,id)
return get
# 利用异常捕获和递归实现网络延迟下的稳定抓取
def find_element_by_xpath(driver,id):
try:
get = driver.find_element_by_xpath(id)
except:
time.sleep(0.2)
get = find_element_by_xpath(driver,id)
return get
def find_element_by_css_selector2(driver,id,ele):
try:
get = driver.find_element_by_css_selector(id)
except:
ele.click()
time.sleep(1)
get = find_element_by_css_selector2(driver,id,ele)
return get
主程序流程：打开平台，自动登录帐号，手动操作通过验证码，然后自动进入高级查询查询成都企业数据，然后执行把日期分段到每段5000条结果内的操作
if __name__ == '__main__':
sys.setrecursionlimit(20000)
#打开数据库连接
conn = pymysql.connect(host = 'localhost',user = "root",passwd = "",db = "企业信息")
Paging_file = open(Paging_path,'a+',encoding='utf8',buffering=1)
#获取游标
cur=conn.cursor()
logfile = open(log_path,'a+',encoding='utf8',buffering=1)
driver = webdriver.Chrome()
driver.maximize_window()
driver.get("https://www.qcc.com/weblogin")
ele_login = find_element_by_class_name(driver,"login-change")
ele_login.click()
ele_login2 = find_element_by_link_text(driver,"密码登录")
ele_login2.click()
inputs = driver.find_elements_by_tag_name("input")
for input in inputs:
if "phone-number"==input.get_attribute("name"):
ele_username = input
if "password"==input.get_attribute("name"):
ele_password = input
ele_username.send_keys("你的帐号")
ele_password.send_keys("你的密码")
inputs =driver.find_elements_by_tag_name("button")
for input in inputs:
#print(input.get_attribute("id")+" | "+ input.get_attribute("name")+" | "+input.get_attribute("class"));
if "btn btn-primary login-btn"==input.get_attribute("class"):
ele_login3 = input
ele_login3.click()
find_element_by_css_selector(driver,"#searchKey")
driver.get("https://www.qcc.com/web/search/advance")
ele_select1 = find_element_by_css_selector(driver,"body > div:nth-child(2) > div.app-search-advance > div.fixed-bottom > div > div.bottom-result > div > a.btn.btn-default.m-r.store-tip-container")
ele_select1.click()
ele_select2 = find_element_by_css_selector(driver,"body > div.app-nmodal.modal.fade.in > div > div > div.modal-body > div > section > ul > div:nth-child(1)")
ele_select2.click()
ele_select3 = find_element_by_link_text(driver,"重新选择")
ele_select3.click()
time.sleep(2)
ele_select4 = find_element_by_xpath(driver,"/html/body/div[1]/div[2]/div[1]/div/div/div[2]/div[5]/div[1]/div[2]/div[8]/label")
ele_select4.click()
inputs =driver.find_elements_by_class_name("ant-calendar-input")
ele_startDate = inputs[0]
ele_endDate = inputs[1]
#getPagingFile(datetime.datetime.strptime("1800-01-01", "%Y-%m-%d"),datetime.datetime.strptime("1980-10-01", "%Y-%m-%d"),ele_startDate,ele_endDate)
#print("结束数量抓取")
Paging_file.close()
Paging_file = open(Paging_path,'r',encoding='utf8')
dateRanges=Paging_file.readlines()
for dateRange in dateRanges:
startDateSTR = dateRange.split()[0]
endDateSTR = dateRange.split()[1]
#insertSQLStrings="INSERT IGNORE INTO `企业信息`.`成都企业信息库`(`企业名称`, `登记状态`, `法定代表人`, `注册资本`, `注册日期`, `纳税人识别号`, `电话`, `邮箱`, `主页`, `注册地址`, `企查查编号`, `工商注册号`, `企业标签`, `企业类型`, `企业地址经度`, `企业地址纬度`, `企业地址城市`, `企业地址区县`, `企业行业标签`, `行业大类`, `行业中类`, `行业小类`, `行业最小类`) VALUES "
ele_temp = driver.find_elements_by_link_text("重置筛选")
if ele_temp == []:
ele_select3 = find_element_by_link_text(driver,"重新选择")
while 1:
#滚动到浏览器顶部
js_top = "var q=document.documentElement.scrollTop=0"
driver.execute_script(js_top)
try:
ele_select3.click()
break
except:
ele_select3 = find_element_by_link_text(driver,"重新选择")
time.sleep(2)
while 1:
try:
ele_select4 = find_element_by_xpath(driver,"/html/body/div[1]/div[2]/div[1]/div/div/div[2]/div[5]/div[1]/div[2]/div[8]/label/label/span[2]")
ele_select4.click()
break
except:
driver.refresh()
time.sleep(1)
inputs =driver.find_elements_by_class_name("ant-calendar-input")
ele_startDate = inputs[0]
ele_endDate = inputs[1]
ele_startDate.send_keys(Keys.CONTROL,'a')
time.sleep(0.2)
ele_startDate.send_keys(Keys.BACK_SPACE)
ele_startDate.send_keys(startDateSTR)
ele_endDate.send_keys(Keys.CONTROL,'a')
time.sleep(0.2)
ele_endDate.send_keys(Keys.BACK_SPACE)
ele_endDate.send_keys(endDateSTR)
while 1:
try:
#滚动到浏览器顶部
js_top = "var q=document.documentElement.scrollTop=0"
driver.execute_script(js_top)
time.sleep(0.3)
ele_temp = find_element_by_css_selector(driver,"body > div:nth-child(2) > div.app-search-advance > div.container.m-t > div > div > div.npanel-body > div:nth-child(4) > span")
time.sleep(0.3)
ele_temp.click()
time.sleep(0.1)
except:
time.sleep(0.2)
print("点击查找失败1")
try:
#点击查找 失败五次额外处理
ele_select5 = find_element_by_css_selector(driver,"body > div:nth-child(2) > div.app-search-advance > div.fixed-bottom > div > div.bottom-result > div > a.btn.btn-primary")
ele_select5.click()
print("即将抓下一批，休眠6秒")
time.sleep(6)
break
except:
time.sleep(0.2)
print("点击查找失败2")
#每页显示40条结果
temp = ""
while temp.find("40")<0:
#ele_select6 = find_element_by_css_selector2(driver,"body > div:nth-child(2) > div.app-search-advance > div > nav > ul > li.size-change > a",ele_select5)
ele_select6 = find_element_by_css_selector(driver,"body > div:nth-child(2) > div.app-search-advance > div > nav > ul > li.size-change > a")
temp = ele_select6.text
time.sleep(0.5)
ele_select6.click()
ele_select7 = find_element_by_css_selector2(driver,"body > div:nth-child(2) > div.app-search-advance > div > nav > ul > li.size-change.open > div > a:nth-child(3)",ele_select6)
ele_select7.click()
time.sleep(0.5)
temp = 0
while 1:
temp+=1
#等待加载出请求的数据结果为止
res=None
trytime = 0
while res==None:
trytime+=1
time.sleep(0.1)
reqs = driver.requests
reqs.reverse()
#reqs[0].response.body.decode
res = getResonseByRequests(reqs,temp)
if trytime>=6:
driver.refresh()
trytime = 0
data = json.loads(res)
insertSQLStrings="INSERT IGNORE INTO `企业信息`.`成都企业信息库`(`企业名称`, `登记状态`, `法定代表人`, `注册资本`, `注册日期`, `纳税人识别号`, `电话`, `邮箱`, `主页`, `注册地址`, `企查查编号`, `工商注册号`, `企业标签`, `企业类型`, `企业地址经度`, `企业地址纬度`, `企业地址城市`, `企业地址区县`, `企业行业标签`, `行业大类`, `行业中类`, `行业小类`, `行业最小类`) VALUES "
if 'Result' in data:
lists = data['Result']
for list in lists:
Name = list['Name']
Name = Name.replace("'","")
KeyNo = list['KeyNo']
No = list['No']
CreditCode = list['CreditCode']
OperName= list['OperName']
ShortStatus= list['ShortStatus']
#毫秒级时间戳转为日期
StartDate= list['StartDate']
timestamp = StartDate
# 转换成localtime
try:
time_local = time.localtime(timestamp/1000)
# 转换成新的时间格式(精确到秒)
dt = time.strftime("%Y-%m-%d %H:%M:%S", time_local)
d = datetime.datetime.fromtimestamp(timestamp/1000)
StartDate = d.strftime("%Y-%m-%d")
except:
#1970年以前日期是另外公式
timestamp = timestamp/1000
StartDate = datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=timestamp+8*3600)
StartDate = StartDate.strftime("%Y-%m-%d")
Address = list['Address']
RegistCapi = list['RegistCapi']
ContactNumber= list['ContactNumber']
Email= list['Email']
strGW= list['GW']
EconKind = list['EconKind']
strX = list['X']
strY= list['Y']
Area= list['Area']
City = ""
if 'City' in Area:
City= Area['City']
County = ""
if 'County' in Area:
County = Area['County']
Industry = list['Industry']
SubIndustry = ""
MiddleCategory = ""
SmallCategory = ""
if 'SubIndustry' in Industry:
SubIndustry= Industry['SubIndustry']
if 'MiddleCategory' in Industry:
MiddleCategory = Industry['MiddleCategory']
if 'SmallCategory' in Industry:
SmallCategory = Industry['SmallCategory']
Industry= Industry['Industry']
Tag= list['Tag']
Tag = Tag.replace("\t"," ")
TagsInfos = list['TagsInfo']
TagsInfosStr =""
if TagsInfos!=None:
for TagsInfo in TagsInfos:
TagsInfo = TagsInfo['n']
TagsInfosStr=TagsInfosStr+TagsInfo+" "
# print(Name)
#这里要用批量插入，不要一条插一次，不然一条就要半秒钟
insertSQLString = "('{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}',{},{},'{}','{}','{}','{}','{}','{}','{}'),".format(Name,ShortStatus,OperName,RegistCapi,StartDate,CreditCode,ContactNumber,Email,strGW,Address,KeyNo,No,Tag,EconKind,strX,strY,City,County,TagsInfosStr,Industry,SubIndustry,MiddleCategory,SmallCategory)
insertSQLString = insertSQLString.replace("\\","")
insertSQLStrings = insertSQLStrings + insertSQLString
insertSQLStrings = insertSQLStrings[:-1]
try:
cur.execute(insertSQLStrings)
#conn.commit()
except Exception as e:
print(insertSQLStrings)
print((str)(datetime.datetime.now())+"抓企查查通过搜索 写入数据库 出现异常："+str(e)+" "+insertSQLStrings)
logfile.write((str)(datetime.datetime.now())+"抓企查查通过搜索 写入数据库 出现异常："+str(e)+" "+insertSQLStrings+"\n")
#根据页数计算这批是否完结,完结后则跳出循环再选新日期
if 'Paging' in data:
lists = data['Paging']
PageSize = lists['PageSize']
PageIndex = lists['PageIndex']
TotalRecords = lists['TotalRecords']
#print(PageSize*PageIndex)
if PageSize*PageIndex>=TotalRecords:
del driver.requests
break
print("准备进下一页"+(str)(temp))
##删除当前请求很重要，否则浏览器打开至今的请求都会累计，最后抓一页都要几万条请求获取速度超慢！
time1 = datetime.datetime.now()
del driver.requests
time2 = datetime.datetime.now()
print(time2-time1)
#这一页解析完成后，开始进入下一页
js_buttom = "var q=document.documentElement.scrollTop=10000"
driver.execute_script(js_buttom)
time.sleep(0.2)
ele_select8 = driver.find_element_by_link_text(">")
time3 = datetime.datetime.now()
print(time3-time2)
#这里加延迟是防止企查查限制
time.sleep(0.5)
ele_select8.click()
#如果因为网络延迟，点了下一页也没动一直停留当前页，做出判定
waittime=0
time33 = datetime.datetime.now()
print(time33-time3)
while 1:
if (waittime>=6):
print("多次尝试进入下一页")
time11 = datetime.datetime.now()
js_buttom = "var q=document.documentElement.scrollTop=10000"
driver.execute_script(js_buttom)
time22 = datetime.datetime.now()
print(time22-time11)
time.sleep(0.2)
ele_select8 = driver.find_element_by_link_text(">")
time.sleep(0.1)
time33 = datetime.datetime.now()
print(time33-time22)
ele_select8.click()
waittime=0
time333 = datetime.datetime.now()
print(time333-time33)
currenPage = driver.find_element_by_class_name("active").text
time4 = datetime.datetime.now()
print(time4-time333)
if (currenPage==(str)(PageIndex)):
time.sleep(1)
print("跳转失败，等待一秒")
waittime+=1
else:
print("已经进下一页"+(str)(temp))
break
conn.commit()
print("抓取完一批，休眠5秒")
#刷新很重要！可以清空浏览器缓存，避免长期执行内存占用累计直至崩溃！
driver.refresh()
time.sleep(5)
# insertSQLStrings = insertSQLStrings[:-1]
# time1 = datetime.datetime.now()
# try:
# cur.execute(insertSQLStrings)
# conn.commit()
# except Exception as e:
# print(insertSQLStrings)
# print((str)(datetime.datetime.now())+"抓企查查通过搜索 写入数据库 出现异常："+str(e)+" "+insertSQLStrings)
# logfile.write((str)(datetime.datetime.now())+"抓企查查通过搜索 写入数据库 出现异常："+str(e)+" "+insertSQLStrings+"\n")
# time2 = datetime.datetime.now()
# print(time2-time1)
# print("本批次抓取插入结束，总用时")
# print(time2-time11)
# print("\n")
Paging_file.close()
conn.close()
logfile.close()这里面有两个关键方法，一个是获取接口返回的时间短内企业数量getCountResonseByRequests，一个是根据日期和日期区间内企业数量来划定最终日期区间方法getPagingFile。
getCountResonseByRequests参数(reqs,startDate,endDate) ，reqs是所有请求参数，通过selniumwire获取的请求头是打开浏览器后的所有headers，所以需要筛选我们需要的请求的最新的headers，startDate,endDate是日期区间的开始与结束日期
#获取request请求列表的返回数据，一定要有返回结果,避免网络延迟无返回结果
def getCountResonseByRequests(reqs,startDate,endDate):
startDate = startDate.replace("-","")
endDate = endDate.replace("-","")
#print(startDate)
for req in reqs:
if ((str)(req.body).find(endDate)>0) and ((str)(req.body).find(startDate)>0):
try:
#driver里的请求参数是对的但不会返回后面页数请求的结果，所以必须取出后面页的header再重新发起post请求
res=requests.post("https://www.qcc.com/api/search/searchCount",data=req.body,headers=req.headers).text
print(req.body)
data = json.loads(res)
if 'Result' not in data:
return 0
count = data['Result']
count = count['Count']
#print(count)
return count
except:
#用异常捕获并延迟等待循环请求返回数据，否则请求有了返回数据还无
time.sleep(1)
getCountResonseByRequests(reqs,startDate,endDate)
getPagingFile参数(startDate,endDate,ele_startDate,ele_endDate)是开始结束日期，与浏览器中开始结束日期输入框元素的位置。
#把结果分为5000页内日期区间
def getPagingFile(startDate,endDate,ele_startDate,ele_endDate):
startDateSTR = (str)(startDate)[:10]
endDateSTR = (str)(endDate)[:10]
if endDate<=lastDate:
ele_startDate.send_keys(Keys.CONTROL,'a')
time.sleep(0.2)
ele_startDate.send_keys(Keys.BACK_SPACE)
ele_startDate.send_keys(startDateSTR)
ele_endDate.send_keys(Keys.CONTROL,'a')
time.sleep(0.2)
ele_endDate.send_keys(Keys.BACK_SPACE)
ele_endDate.send_keys(endDateSTR)
#temp = find_element_by_css_selector(driver,"body > div:nth-child(2) > div.app-search-advance > div.container.m-t > div > div > div.npanel-body > div:nth-child(5) > span")
#temp.click()
count=None
while count==None:
time.sleep(1)
reqs = driver.requests
reqs.reverse()
count = getCountResonseByRequests(reqs,startDateSTR,endDateSTR)
#2003年以前，以30天为分段；2003-2015以5天为分段；2015以后，每天为分段。期间如果多天分段的大于5000条则进一步拆分为每天分段，如果每天也大于5000则记录到日志中跳过
if endDate5000:
#当区间断大于5000条结果，则按每天来分段，若每天里还有大于5000的，记录并跳过
if (endDate != startDate):
endDate=startDate
getPagingFile(startDate,endDate,ele_startDate,ele_endDate)
else:
print(startDateSTR+"-"+endDateSTR+"超过5000，请手动查看处理")
Paging_logfile.write(startDateSTR+"-"+endDateSTR+"超过5000，请手动查看处理"+"\n")
startDate = endDate+datetime.timedelta(days=1)
endDate=startDate
getPagingFile(startDate,endDate,ele_startDate,ele_endDate)
else:
print(startDateSTR+" "+endDateSTR+" "+(str)(count))
Paging_file.write(startDateSTR+" "+endDateSTR+" "+(str)(count)+"\n")
startDate = endDate+datetime.timedelta(days=1)
endDate=endDate+datetime.timedelta(days=31)
getPagingFile(startDate,endDate,ele_startDate,ele_endDate)
elif endDate5000:
#当区间断大于5000条结果，则按每天来分段，若每天里还有大于5000的，记录并跳过
if (endDate != startDate):
endDate=startDate
getPagingFile(startDate,endDate,ele_startDate,ele_endDate)
else:
print(startDateSTR+"-"+endDateSTR+"超过5000，请手动查看处理")
Paging_logfile.write(startDateSTR+"-"+endDateSTR+"超过5000，请手动查看处理"+"\n")
startDate = endDate+datetime.timedelta(days=1)
endDate=startDate
getPagingFile(startDate,endDate,ele_startDate,ele_endDate)
else:
print(startDateSTR+" "+endDateSTR+" "+(str)(count))
Paging_file.write(startDateSTR+" "+endDateSTR+" "+(str)(count)+"\n")
startDate = endDate+datetime.timedelta(days=1)
endDate=endDate+datetime.timedelta(days=6)
getPagingFile(startDate,endDate,ele_startDate,ele_endDate)
else:
if count<1000:
endDate = endDate+datetime.timedelta(days=1)
getPagingFile(startDate,endDate,ele_startDate,ele_endDate)
elif count>5000:
print(startDateSTR+"-"+endDateSTR+"超过5000，请手动查看处理")
Paging_logfile.write(startDateSTR+"-"+endDateSTR+"超过5000，请手动查看处理"+"\n")
startDate = endDate+datetime.timedelta(days=1)
endDate=startDate
getPagingFile(startDate,endDate,ele_startDate,ele_endDate)
else:
print(startDateSTR+" "+endDateSTR+" "+(str)(count))
Paging_file.write(startDateSTR+" "+endDateSTR+" "+(str)(count)+"\n")
#如果抓取了所有日期结果分段，则结束数量抓取方法
if endDate==lastDate:
return
startDate = endDate+datetime.timedelta(days=1)
endDate=startDate
getPagingFile(startDate,endDate,ele_startDate,ele_endDate)
else:
getPagingFile(startDate,lastDate,ele_startDate,ele_endDate)
这个过程完成后后，我们就可以获得写入txt的日期分段，每行数据是日期区间的开始日期、结束日期、日期区间内企业数量
然后再开始按遍历所有日期区间开始一页页抓取调用原生接口searchmulti，这里写一个解析该接口结果的方法
#获取request请求列表的返回数据，一定要有返回结果,避免网络延迟无返回结果
def getResonseByRequests(reqs,page):
strPage = "\"pageIndex\":"+(str)(page)
for req in reqs:
if ((str)(req.body).find("\"pageSize\":40")>0) and (str)(req.body).find(strPage)>0:
try:
#driver里的请求参数是对的但不会返回后面页数请求的结果，所以必须取出后面页的header再重新发起post请求
res=requests.post("https://www.qcc.com/api/search/searchMulti",data=req.body,headers=req.headers).text
# res = req.response.body
# res=gzip.decompress(res).decode("utf-8")
# print(res)
print(req.body)
#print("获取数据中")
#print(res)
return res
except:
#用异常捕获并延迟等待循环请求返回数据，否则请求有了返回数据还无
time.sleep(1)
getResonseByRequests(reqs,page)
主程序里开始解析每个日期区间的结果并写入数据库
Paging_file = open(Paging_path,'r',encoding='utf8')
dateRanges=Paging_file.readlines()
for dateRange in dateRanges:
startDateSTR = dateRange.split()[0]
endDateSTR = dateRange.split()[1]
ele_temp = driver.find_elements_by_link_text("重置筛选")
if ele_temp == []:
#滚动到浏览器顶部
js_top = "var q=document.documentElement.scrollTop=0"
driver.execute_script(js_top)
ele_select3 = find_element_by_link_text(driver,"重新选择")
ele_select3.click()
time.sleep(2)
ele_select4 = find_element_by_xpath(driver,"/html/body/div[1]/div[2]/div[1]/div/div/div[2]/div[6]/div[1]/div[2]/div[8]/label")
ele_select4.click()
inputs =driver.find_elements_by_class_name("ant-calendar-input")
ele_startDate = inputs[0]
ele_endDate = inputs[1]
ele_startDate.send_keys(Keys.CONTROL,'a')
time.sleep(0.2)
ele_startDate.send_keys(Keys.BACK_SPACE)
ele_startDate.send_keys(startDateSTR)
ele_endDate.send_keys(Keys.CONTROL,'a')
time.sleep(0.2)
ele_endDate.send_keys(Keys.BACK_SPACE)
ele_endDate.send_keys(endDateSTR)
ele_temp = find_element_by_css_selector(driver,"body > div:nth-child(2) > div.app-search-advance > div.container.m-t > div > div > div.npanel-body > div:nth-child(5) > span")
ele_temp.click()
#点击查找
ele_select5 = find_element_by_css_selector(driver,"body > div:nth-child(2) > div.app-search-advance > div.fixed-bottom > div > div > a.btn.btn-primary")
ele_select5.click()
#每页显示40条结果
temp = ""
while temp.find("40")<0:
ele_select6 = find_element_by_css_selector2(driver,"body > div:nth-child(2) > div.app-search-advance > div > nav > ul > li.size-change > a",ele_select5)
temp = ele_select6.text
ele_select6.click()
ele_select7 = find_element_by_css_selector2(driver,"body > div:nth-child(2) > div.app-search-advance > div > nav > ul > li.size-change.open > div > a:nth-child(3)",ele_select6)
ele_select7.click()
time.sleep(0.5)
temp = 0
while 1:
temp+=1
#等待加载出请求的数据结果为止
res=None
while res==None:
time.sleep(1)
reqs = driver.requests
reqs.reverse()
res = getResonseByRequests(reqs,temp)
data = json.loads(res)
if 'Result' in data:
lists = data['Result']
for list in lists:
Name = list['Name']
KeyNo = list['KeyNo']
No = list['No']
CreditCode = list['CreditCode']
OperName= list['OperName']
ShortStatus= list['ShortStatus']
#毫秒级时间戳转为日期
StartDate= list['StartDate']
timestamp = StartDate
# 转换成localtime
try:
time_local = time.localtime(timestamp/1000)
# 转换成新的时间格式(精确到秒)
dt = time.strftime("%Y-%m-%d %H:%M:%S", time_local)
d = datetime.datetime.fromtimestamp(timestamp/1000)
StartDate = d.strftime("%Y-%m-%d")
except:
#1970年以前日期是另外公式
timestamp = timestamp/1000
StartDate = datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=timestamp+8*3600)
StartDate = StartDate.strftime("%Y-%m-%d")
Address = list['Address']
RegistCapi = list['RegistCapi']
ContactNumber= list['ContactNumber']
Email= list['Email']
strGW= list['GW']
EconKind = list['EconKind']
strX = list['X']
strY= list['Y']
Area= list['Area']
City = ""
if 'City' in Area:
City= Area['City']
County = ""
if 'County' in Area:
County = Area['County']
Industry = list['Industry']
SubIndustry = ""
MiddleCategory = ""
SmallCategory = ""
if 'SubIndustry' in Industry:
SubIndustry= Industry['SubIndustry']
if 'MiddleCategory' in Industry:
MiddleCategory = Industry['MiddleCategory']
if 'SmallCategory' in Industry:
SmallCategory = Industry['SmallCategory']
Industry= Industry['Industry']
Tag= list['Tag']
Tag = Tag.replace("\t"," ")
TagsInfos = list['TagsInfo']
TagsInfosStr =""
if TagsInfos!=None:
for TagsInfo in TagsInfos:
TagsInfo = TagsInfo['n']
TagsInfosStr=TagsInfosStr+TagsInfo+" "
# print(Name)
try:
insertSQLString = "INSERT IGNORE INTO `企业信息`.`成都企业信息库`(`企业名称`, `登记状态`, `法定代表人`, `注册资本`, `注册日期`, `纳税人识别号`, `电话`, `邮箱`, `主页`, `注册地址`, `企查查编号`, `工商注册号`, `企业标签`, `企业类型`, `企业地址经度`, `企业地址纬度`, `企业地址城市`, `企业地址区县`, `企业行业标签`, `行业大类`, `行业中类`, `行业小类`, `行业最小类`) VALUES ('{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}',{},{},'{}','{}','{}','{}','{}','{}','{}');".format(Name,ShortStatus,OperName,RegistCapi,StartDate,CreditCode,ContactNumber,Email,strGW,Address,KeyNo,No,Tag,EconKind,strX,strY,City,County,TagsInfosStr,Industry,SubIndustry,MiddleCategory,SmallCategory)
cur.execute(insertSQLString)
conn.commit()
except Exception as e:
print(insertSQLString)
print((str)(datetime.datetime.now())+"通过搜索 写入数据库 出现异常："+str(e)+" "+insertSQLString)
logfile.write((str)(datetime.datetime.now())+"通过搜索 写入数据库 出现异常："+str(e)+" "+insertSQLString+"\n")
#根据页数计算这批是否完结,完结后则跳出循环再选新日期
if 'Paging' in data:
lists = data['Paging']
PageSize = lists['PageSize']
PageIndex = lists['PageIndex']
TotalRecords = lists['TotalRecords']
#print(PageSize*PageIndex)
if PageSize*PageIndex>TotalRecords:
break
#这一页解析完成后，开始进入下一页
ele_select8 = find_element_by_link_text(driver,">")
ele_select8.click()
Paging_file.close()
conn.close()
logfile.close()至此完成抓取，全过程基本自动化，只有脚本启动后的登录过程需要手动通过验证码，同理可以抓取其他省市、其他限定条件的数据。