利用python爬取全国疫情信息并绘制中国疫情图
利用python爬取全国疫情信息并绘制中国疫情图
版权声明：本文为CSDN博主「ymx12349」的原创文章，遵循CC 4.0 BY-SA版权协议。本次转载是在原作的基础上进行了代码调整，本次代码在树莓派4B，python3.7下运行全部跑通。
原文链接：https://blog.csdn.net/ymx12349/article/details/113191611
数据爬取
1、进入腾讯疫情实时更新首页，网址为：https://news.qq.com/zt2020/page/feiyan.htm#/
2、寻找数据所在的API，https://api.inews.qq.com/newsqa/v1/query/pubished/daily/list?province=省份
3、构建一个excle表格，内容为中国各个省份的名称。
4、爬取结果为json数据，通过解析，得到想要的内容。
疫情地图绘制
1、利用python的pyecharts库来绘制中国地图
2、本文以中国累计确诊人数为例
修改部分：
1. 因为了api接口返回的数据是包括了全部的日期的，因此，保存数据时，只留了最近一天的数据。
2. api接口调用的时候，有时会被服务器拒绝，因此增加了sleep和http的报文头。并增加了异常情况下的重复调用。
import json
import requests
import pandas as pd
import time
shengfen=pd.read_excel(r'/home/pi/Documents/pythonspace/dataproc/省份.xls',index_col=0)
data1=[]
# print(shengfen)
today=time.localtime()
headers = {'User-Agent':r'Mozilla/5.0 (X11; Linux armv7l) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.197 Safari/537.36'}
print("今天是%d年%d月%d日" %(today.tm_year, today.tm_mon, today.tm_mday))
for j in range(34):
url=r'https://api.inews.qq.com/newsqa/v1/query/pubished/daily/list?province='+shengfen.iloc[j,0]
print(url)
time.sleep(3)
# html = requests.get(url, headers = headers)
while True:
try:
# s = requests.session()
# s.keep_alive = False
print(headers)
html = requests.get(url, headers = headers)
print(html.headers)
# html=requests.get(r'https://api.inews.qq.com/newsqa/v1/query/pubished/daily/list?province=上海')
break
except:
print("Connection refused by the server..")
print("Let me sleep for 5 seconds")
print("ZZzzzz...")
time.sleep(5)
print("Was a nice sleep, now let me continue...")
continue
text = html.text
print(text[1:100])
maxdate = int(20010101)
data=json.loads(html.text)['data']
for i in data:
data2={}
if i['year']==today.tm_year and int(i['date'][0:2])==today.tm_mon and int(i['date'][3:5])==today.tm_mday:
data2['日期']=str(i['year'])+'年'+i['date'][0:2]+'月'+i['date'][3:5]+'日'
data2['省份']=i['province']
data2['确诊']=i['confirm']
data2['死亡']=i['dead']
data2['治愈']=i['heal']
data2['新增确诊']=i['newConfirm']
data2['新增治愈']=i['newHeal']
data2['新增死亡']=i['newDead']
# data1.append(data2)
break
if int(i['year']) * 10000 + int(i['date'][0:2]) * 100 + int(i['date'][3:5]) > maxdate :
maxdate = int(i['year']) * 10000 + int(i['date'][0:2]) * 100 + int(i['date'][3:5])
data2['日期']=str(i['year'])+'年'+i['date'][0:2]+'月'+i['date'][3:5]+'日'
data2['省份']=i['province']
data2['确诊']=i['confirm']
data2['死亡']=i['dead']
data2['治愈']=i['heal']
data2['新增确诊']=i['newConfirm']
data2['新增治愈']=i['newHeal']
data2['新增死亡']=i['newDead']
data1.append(data2)
df=pd.DataFrame(data1)
df.to_excel(r'/home/pi/Documents/pythonspace/dataproc/全国疫情历史数据2.xlsx')
1. 改panda的数据读取为使用openpyxl
2.原代码的数据是在程序里面写死的，改成从excel读取。
import pandas as pd
import numpy as np
from pyecharts.charts import Map,Page,Timeline,Bar
from pyecharts import options as opts
from openpyxl import Workbook, load_workbook
import xlrd
import time
# data = pd.read_excel(r'/home/pi/Documents/pythonspace/dataproc/全国疫情历史数据2.xlsx')
# data1 = data.loc[:, ['省份', '确诊']]
# data1=data.groupby('省份')
# print(data1['确诊'])
# print(data1)
z = [] #列表，付给map
packagetype='openpyxl' # value: openpyxl xlrd
if 'openpyxl' == packagetype:
#使用openpyxl处理
wb = load_workbook(r'/home/pi/Documents/pythonspace/dataproc/全国疫情历史数据2.xlsx', data_only = True)
#ws = wb.sheetnames[0]
ws = wb.get_sheet_by_name('Sheet1')
for row in ws.rows:
if row[2].value == '省份':
continue
zz=[]
zz.append(row[2].value)
zz.append(row[3].value)
print(zz)
z.append(zz)
print(z)
else:
#使用xlrd处理
book=xlrd.open_workbook(r'/home/pi/Documents/pythonspace/dataproc/全国疫情历史数据2.xls', 0)
sh=book.sheet_by_index(0)
cols = sh.ncols
rows = sh.nrows
print('cols=',cols, 'rows=',rows)
#逐行将数据转入列表中
for r in range(rows): # cols and rows start from 0
#第一行是标题，跳过
if 0 == r :
continue
zz=[] #列表，存储省份、确诊数据
zz.append(sh.cell_value(rowx=r,colx=2)) #省份
zz.append(sh.cell_value(rowx=r,colx=3)) #确诊
print(zz)
z.append(zz)
# z=[['上海', 307030],['云南', 70636], ['内蒙古', 83305],['北京', 279327],['台湾', 164154],['吉林', 50358],['四川', 231099],['天津', 77764],['宁夏', 26343],['安徽', 350955],['山东', 283914],['山西', 68676],['广东', 608068],['广西', 91166],['新疆', 179791],['江苏', 233516],['江西', 331116],['河北', 131779],['河南', 452710],['浙江', 452542],['海南', 60422],['湖北', 23701957],['湖南', 362204],['澳门', 14446],['甘肃', 54198],['福建', 139069],['西藏', 359],['贵州', 51781],['辽宁', 76997],['重庆', 207060],['陕西', 128029],['青海', 6449],['香港', 1222340],['黑龙江', 304942]]#这里一定要把数据重新写一遍，不然会报错
m2=(Map()
.add("全国疫情累计确诊人数",z,"china",is_map_symbol_show=False)
.set_global_opts(
title_opts=opts.TitleOpts(title="Map-VisualMap(分段型)"),
visualmap_opts=opts.VisualMapOpts(max_=100000,min_=0,is_piecewise=True,
pieces=[
{"min":20000,"label":"大于20000"},
{"min":2000,"max":19999,"label":"大于19999"},
{"min":1000,"max":1999,"label":"大于999"},
{"min":400,"max":999,"label":"大于399"},
{"min":200,"max":399,"label":"大于199"},
{"min":100,"max":199,"label":"大于100"},
{"min":0,"max":99,"label":"大于0"},
])
))
st=time.localtime()
#按时间截拼接url
strtime=str(st.tm_year)+str(st.tm_mon)+str(st.tm_mday)+str(st.tm_hour).ljust(2,'0')+str(st.tm_min).ljust(2,'0')+str(st.tm_sec).ljust(2,'0')
disurl=r'/home/pi/Documents/pythonspace/dataproc/yiqingmap'+strtime+'.html'
m2.render(disurl)
| 序号 | 省份 |
| 1 | 上海 |
| 2 | 云南 |
| 3 | 内蒙古 |
| 4 | 北京 |
| 5 | 台湾 |
| 6 | 吉林 |
| 7 | 四川 |
| 8 | 天津 |
| 9 | 宁夏 |
| 10 | 安徽 |
| 11 | 山东 |
| 12 | 山西 |
| 13 | 广东 |
| 14 | 广西 |
| 15 | 新疆 |
| 16 | 江苏 |
| 17 | 江西 |
| 18 | 河北 |
| 19 | 河南 |
| 20 | 浙江 |
| 21 | 海南 |
| 22 | 湖北 |
| 23 | 湖南 |
| 24 | 澳门 |
| 25 | 甘肃 |
| 26 | 福建 |
| 27 | 西藏 |
| 28 | 贵州 |
| 29 | 辽宁 |
| 30 | 重庆 |
| 31 | 陕西 |
| 32 | 青海 |
| 33 | 香港 |
| 34 | 黑龙江 |