import json
import re
import urllib.request
import urllib.parse
from bs4import BeautifulSoup
def main():
url ='http://js.8684.cn/citys/city_boxInf.min.js'
#按照拼接获取城市
request = get_city_request(url)
#获取响应的对象
response = get_city_response(request)
#获得第一层数据城市
def get_city_request(url):
headers = {
'User - Agent':'Mozilla / 5.0(Windows NT 6.1;WOW64) AppleWebKit / 537.36(KHTML, likeGecko) Chrome / 67.0.3396.99Safari / 537.36'
}
request = urllib.request.Request(url=url,headers=headers)
return request
#获得响应的对象
def get_city_response(request):
response = urllib.request.urlopen(request)
data = response.read().decode("utf8").split("=")
res = []
#获取汉字版的数字
for iin range(len(data)):
res.append(data[2])
html = res[0].split(",")
for iin html:
# print(i)
#通过正则匹配拼接
a = re.findall(r'(\w+):',i)
print('正在爬取%s' %a)
#创建每个省的文件夹
fp =open(a[0]+'.txt','w',encoding="utf8")
#拼接url
url ='http://'+a[0]+'.8684.cn'
#调用请求对象函数
request = get_city_request(url)
#调用第二层请求对象函数
response = get_response(request)
data = get_data(response,url)
print(msg_list)
for iin msg_list:
print(type(i))
fp.write(str(i))
print("%s爬取结束" %a)
#调用第二层响应的对象
def get_response(request):
try:
response = urllib.request.urlopen(request)
return response
except:
response=''
return response
#获取二层数据
def get_data(response,url):
try:
#实例化一个soup
data = response.read().decode("utf8")
soup = BeautifulSoup(data,'lxml')
#获取以数字开头的公交车信息
html1 = soup.select('.bus_kt_r1 > a')
html2 = soup.select('.bus_kt_r2 > a')
#将两个获取的列表相加
html = html1+html2
#遍历循环HTML文件里面的数据
for iin html:
#拼接路径
print(url)
url1 = url + i['href']
print('url',url1)
#调用获取请求参数的函数
request = get_city_request(url1)
#调用第二层获取响应数据的函数
response = get_response(request)
#调用第三层的数据
data = get_data_san(response,url)
except:
print("过去")
data=""
#获取三层数据
def get_data_san(response,url):
global msg_list
msg_list = []
data = response.read().decode("utf8")
#实例化一个soup
soup = BeautifulSoup(data,'lxml')
html = soup.select("#con_site_1 > a")
for iin html:
#先截取url，在拼接路径
url = url.split("cn")[0] +'cn' + i['href']
# print(url)
#调用获取请求对象
request = get_city_request(url)
#调用第二层的响应对象
response = get_response(request)
#调用第四层数据
text = get_data_si(response)
msg_list.append(text)
print('msg',msg_list)
#获取第四层数据
def get_data_si(res):
data = res.read().decode("utf8")
#实例化一个soup
soup = BeautifulSoup(data,'lxml')
#获取具体的信息
#获得公交名字
bus_name = soup.select(".bus_i_t1 > h1")[0].text
#获得公交运行时间
bus_run_time = soup.select('.bus_i_t4')[0].text
#获得公交的票价
bus_fares = soup.select('.bus_i_t4')[1].text.split(" ")[-1]
#获得公交的运行公司
bus_company = soup.select('.bus_i_t4')[2].text
#获得更新时间
bus_msg_update_time = soup.select('.bus_i_t4')[-1].text.split("：")[-1]
#获取正向行驶的路线
up_run = soup.select(".bus_line_site")[0]
up = up_run.select('div > div > a')
up_temp = []
for iin up:
up_temp.append(i.text)
# print(up_temp)
try:
#获取反向行驶路线
down_run = soup.select(".bus_line_site")[1]
down = down_run.select('div > div > a')
down_temp = []
for iin down:
down_temp.append(i.text)
# print(down_temp)
except:
down_temp = []
items = {
'公交名字':bus_name,
'运行时间':bus_run_time,
'票价':bus_fares,
'公交运行公司':bus_company,
'路线更新时间':bus_msg_update_time,
'正向行驶路线':up_temp,
'反向行驶路线':down_temp
}
# print(items)
return items
if __name__ =='__main__':
main()