空气质量数据下载
从pm25.in网站下载全国各个城市、各个站点逐小时空气质量数据。Github地址：https://github.com/lixiang-ucas/PM25
使用Urllib2和BeautifulSoup库进行数据抓取
getUrlRespHtml是下载函数，设置了一些HTTP头
getCities获取城市列表
getCurrentHour获取当前一个小时的数据
__name__入口函数，每小时执行一次getCurrentHour
# -*- coding: utf-8 -*-
"""
Created on Tue Jan 26 14:28:14 2016
@author: lixiang
"""
import urllib2
import socket
import time
from bs4 import BeautifulSoup #besutifulsoup的第三版
import sys
reload(sys)
sys.setdefaultencoding( "utf-8" )
sleep_time=30
def getUrlRespHtml(url):
heads = {'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Charset':'GB2312,utf-8;q=0.7,*;q=0.7',
'Accept-Language':'zh-cn,zh;q=0.5',
'Cache-Control':'max-age=0',
'Connection':'keep-alive',
'Host':'John',
'Keep-Alive':'115',
'Referer':url,
'User-Agent':'Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.14) Gecko/20110221 Ubuntu/10.10 (maverick) Firefox/3.6.14'}
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor())
urllib2.install_opener(opener)
req = urllib2.Request(url)
opener.addheaders = heads.items()
# respHtml = opener.open(req).read()
# return respHtml.decode('gbk').encode('utf-8')
try:
respHtml = opener.open(req).read()
except urllib2.HTTPError, e:
print e.code
return respHtml
def getCities():
site='http://pm25.in'
html = getUrlRespHtml(site)
soup = BeautifulSoup(html)
hrefs = soup.find("div",{"class":"all"}).find_all('a')
cs=[]
for href in hrefs:
c=href['href']
c=c[1:len(c)]
cs.append(c)
# cs=cs[0:10]
return list(set(cs))
def getCurrentHour():
socket.setdefaulttimeout(60) # 60 秒钟后超时
file_c = open('city.txt','w')
file_s = open('site.txt','w')
# cities=['beijing','tianjin','shanghai']
cities=getCities()
print 'city count is：'+str(len(cities))
# city='tianjin'
for city in cities:
ind=cities.index(city)
if ind%50==0:
time.sleep(sleep_time)
site ='http://pm25.in/' + city + '.html'
print str(ind)+' '+site
# html = urllib2.urlopen(site)
html = getUrlRespHtml(site)
soup = BeautifulSoup(html)
#时间
dt = soup.find('div',{'class':'live_data_time'}).find('p').string
dt = dt.split('：')[1]
#城市数据
cityAQI = soup.find_all("div",{"class":"value"})
file_c.write(city+'\t')
file_c.write(dt+'\t')
for ind_aqi in range(len(cityAQI)-1):
aqi=cityAQI[ind_aqi]
temp='-'
if aqi.string!=None:
file_c.write(aqi.string.split()[0]+'\t')
else:
file_c.write(temp+'\t')
file_c.write('\n')
#各个站点数据
trs=soup.find_all('tr')
for ind_s in range(1,len(trs)):
file_s.write(city+'\t')
file_s.write(dt+'\t')
tr=trs[ind_s]
siteAQI=tr.find_all('td')
for aqi in siteAQI:
temp='-'
if aqi.string!=None:
file_s.write(aqi.string.split()[0]+'\t')
else:
file_s.write(temp+'\t')
file_s.write('\n')
time.sleep(sleep_time)
file_c.close()
file_s.close()
if __name__ == '__main__':
while True:
getCurrentHour()
time.sleep(3600)