Python编程-使用爬虫程序爬取肺炎疫情数据-2 爬取给定国家的历史疫情信息
文字懒得写了，下面是代码
import json
import easygui
import xlwt
import requests
import datetime
def getDateList():
# 创建从2020-1-20到目前位置的日期序列
start_date = datetime.datetime(2020, 1, 20)
end_date = datetime.datetime.now()
date_list_all = []
for i in range((end_date.date() - start_date.date()).days + 1):
date_list_all.append(start_date + datetime.timedelta(days=i))
return date_list_all, start_date
def getChinaDataAndSave(workbook, china_day_list, daily_new_add_history, date_list_all, start_date, sheet_name = "中国"):
print("开始爬取 %s 的疫情数据... ... " % sheet_name)
# 创建一个worksheet
worksheet = workbook.add_sheet(sheet_name)
# 将列标题写入excel
current_row_index = 0 # 记录当前所写入数据的行号
for i, str_col in enumerate(['date', 'confirm', 'suspect', 'dead', 'heal', 'nowConfirm', 'nowSevere', "importedCase", 'deadRate', 'healRate', "全国新增", '湖北新增', "全国(除湖北)新增"]):
worksheet.write(current_row_index, i, str_col) # 参数对应 行, 列, 值
worksheet.col(i).width = 150 * 20 # 设置excel中第A列的宽度（方便日期数据展示）
current_row_index += 1
# 往excel中写入日期
style = xlwt.XFStyle()
style.num_format_str = 'YYYY/MM/DD'
first_data_date = datetime.datetime.strptime('2020.' + daily_new_add_history[0]['date'], "%Y.%m.%d") # 爬取到疫情数据的最早日期
delta_time = (first_data_date - start_date).days
# 2020-1-20 到 first_data_date(不含后者） 之间未爬取到疫情数据，下面将这些时间数据写入excel
for i in range(delta_time):
worksheet.write(current_row_index, 0, date_list_all[current_row_index - 1], style)
current_row_index += 1
# china_day_list 和 daily_new_add_history中的数据开始日期不一致，通过断点调试可知china_day_list是从2020.1.13开始的， daily_new_add_history是从2020.1.20开始的
china_day_list = [i for i in china_day_list if (datetime.datetime.strptime('2020.' + i['date'], "%Y.%m.%d") - start_date).days >= 0]
# 提取海外数据信息，并写入excel
for china_day_i, daily_new_add_history_i in zip(china_day_list, daily_new_add_history):
worksheet.write(current_row_index, 0, datetime.datetime.strptime('2020.' + china_day_i["date"], "%Y.%m.%d"), style)
worksheet.write(current_row_index, 1, china_day_i['confirm'])
worksheet.write(current_row_index, 2, china_day_i['suspect'])
worksheet.write(current_row_index, 3, china_day_i['dead'])
worksheet.write(current_row_index, 4, china_day_i['heal'])
worksheet.write(current_row_index, 5, china_day_i['nowConfirm'])
worksheet.write(current_row_index, 6, china_day_i['nowSevere'])
worksheet.write(current_row_index, 7, china_day_i['importedCase'])
worksheet.write(current_row_index, 8, china_day_i['deadRate'])
worksheet.write(current_row_index, 9, china_day_i['healRate'])
worksheet.write(current_row_index, 10, daily_new_add_history_i['country'])
worksheet.write(current_row_index, 11, daily_new_add_history_i['hubei'])
worksheet.write(current_row_index, 12, daily_new_add_history_i['notHubei'])
current_row_index += 1
print("%s 的疫情数据爬取完毕" % sheet_name)
def getURLContentGet(url):
headers = {
'user-agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Mobile Safari/537.36'
}
response = requests.get(url, headers=headers)
return response.status_code, response
def getForeignCountryList(foreign_all_data_info):
'''
:param foreign_all_data_info: dict对象，foreign_all_data_info['foreignList']是一个list对象，每一个对象是一个国家的疫情信息
:return:
'''
foreignCountryList = []
foreignList = foreign_all_data_info['foreignList']
for country in foreignList:
foreignCountryList.append(country['name'])
return foreignCountryList
def getCountryEpidemiSituationInfo(workbook, country_list):
date_list_all, start_date = getDateList()
for index, country_i in enumerate(country_list):
print("开始爬取 %s 的疫情数据... ... " % country_i)
url = 'https://api.inews.qq.com/newsqa/v1/query/pubished/daily/list?country=%s' % country_i
status_code, data = getURLContentGet(url)
if status_code != 200:
print("%s数据爬取失败,状态码%d" % (country_i, status_code))
sys.exit()
data = data.json()["data"]
if data == None:
print("\n\n%s数据爬取数据为空\n\n" % country_i)
break
# 创建一个worksheet
worksheet = workbook.add_sheet(country_i)
worksheet.col(0).width = 128 * 20 # 设置excel中第A列的宽度（方便日期数据展示）
current_row_index = 0 # 记录当前所写入数据的行号
# 将列标题写入excel
for i, str_col in enumerate(['date', 'confirm_add', 'confirm', 'heal', 'dead', 'suspect']):
worksheet.write(current_row_index, i, str_col) # 参数对应 行, 列, 值
current_row_index += 1
# 往excel中写入日期
style = xlwt.XFStyle()
style.num_format_str = 'YYYY/MM/DD'
first_data_date = datetime.datetime.strptime('2020.' + data[0]['date'], "%Y.%m.%d") # 爬取到疫情数据的最早日期
delta_time = (first_data_date - start_date).days
# 2020-1-20 到 first_data_date(不含后者） 之间未爬取到疫情数据，下面将这些时间数据写入excel
for i in range(delta_time):
worksheet.write(current_row_index, 0, date_list_all[current_row_index - 1], style)
current_row_index += 1
# 将抓取到的疫情数据写入excel
for data_i in data:
worksheet.write(current_row_index, 0, datetime.datetime.strptime('2020.' + data_i['date'], "%Y.%m.%d"), style)
worksheet.write(current_row_index, 1, data_i['confirm_add'])
worksheet.write(current_row_index, 2, data_i['confirm'])
worksheet.write(current_row_index, 3, data_i['heal'])
worksheet.write(current_row_index, 4, data_i['dead'])
worksheet.write(current_row_index, 5, data_i['suspect'])
current_row_index += 1
def getGlobalDataAndSave(workbook, all_history, date_list_all, start_date, sheet_name = '海外'):
print("开始爬取 %s 的疫情数据... ... " % sheet_name)
# 创建一个worksheet
worksheet = workbook.add_sheet(sheet_name)
worksheet.col(0).width = 128 * 20 # 设置excel中第A列的宽度（方便日期数据展示）
# 将列标题写入excel
current_row_index = 0 # 记录当前所写入数据的行号
for i, str_col in enumerate(['date', 'confirm', 'dead', 'heal', 'newAddConfirm', 'deadRate', 'healRate']):
worksheet.write(current_row_index, i, str_col) # 参数对应 行, 列, 值
current_row_index += 1
# 往excel中写入日期
style = xlwt.XFStyle()
style.num_format_str = 'YYYY/MM/DD'
first_data_date = datetime.datetime.strptime('2020.' + all_history[0]['date'], "%Y.%m.%d") # 爬取到疫情数据的最早日期
delta_time = (first_data_date - start_date).days
# 2020-1-20 到 first_data_date(不含后者） 之间未爬取到疫情数据，下面将这些时间数据写入excel
for i in range(delta_time):
worksheet.write(current_row_index, 0, date_list_all[current_row_index - 1], style)
current_row_index += 1
# 提取海外数据信息，并写入excel
for daily_history_i in all_history:
worksheet.write(current_row_index, 0, datetime.datetime.strptime('2020.' + daily_history_i["date"], "%Y.%m.%d"),
style)
worksheet.write(current_row_index, 1, daily_history_i["all"]['confirm'])
worksheet.write(current_row_index, 2, daily_history_i["all"]['dead'])
worksheet.write(current_row_index, 3, daily_history_i["all"]['heal'])
worksheet.write(current_row_index, 4, daily_history_i["all"]['newAddConfirm'])
worksheet.write(current_row_index, 5, daily_history_i["all"]['deadRate'])
worksheet.write(current_row_index, 6, daily_history_i["all"]['healRate'])
current_row_index += 1
print("%s 的疫情数据爬取完毕" % sheet_name)
def getArticleInfoFromGivedList(workbook, article_list, sheet_name):
# 创建一个worksheet
worksheet = workbook.add_sheet(sheet_name)
if len(article_list) > 0:
current_row_index = 0
#设置列宽
worksheet.col(0).width = 256 * 20
worksheet.col(1).width = 128 * 20
worksheet.col(2).width = 512 * 20
worksheet.col(3).width = 512 * 20
worksheet.col(4).width = 2048 * 20
# 将列标题写入excel
for i, str_col in enumerate(['publish_time', 'media', 'title', 'url', 'desc']):
worksheet.write(current_row_index, i, str_col) # 参数对应 行, 列, 值
current_row_index += 1
for article_i in article_list[-1:- len(article_list) - 1 : -1]:
worksheet.write(current_row_index, 0, article_i["publish_time"])
worksheet.write(current_row_index, 1, article_i["media"])
worksheet.write(current_row_index, 2, article_i["title"])
worksheet.write(current_row_index, 3, article_i["url"])
worksheet.write(current_row_index, 4, article_i["desc"])
current_row_index += 1
if __name__ == "__main__":
foreign_url = 'https://view.inews.qq.com/g2/getOnsInfo?name=disease_foreign'
_, foreign_response = getURLContentGet(foreign_url)
foreign_json_data = json.loads(foreign_response.text)
foreign_all_data = json.loads(foreign_json_data['data'])
foreign_country_list = getForeignCountryList(foreign_all_data)
#foreign_country_list = ["美国", '意大利','丹麦', '日本本土', '韩国', '加拿大', '德国', '英国']
countries_and_regions = ['中国', "海外"]
countries_and_regions.extend(foreign_country_list)
choice = easygui.multchoicebox("\n\t\t请选择要爬取疫情数据的国家和地区：\t\t\n", title="Python爬虫程序", choices=countries_and_regions,
preselect=[0, 1])
# 获取日期列表
date_list_all, start_date = getDateList()
# 创建一个workbook 设置编码
workbook = xlwt.Workbook(encoding='utf-8')
if "中国" in choice:
del choice[choice.index("中国")]
china_url = 'https://view.inews.qq.com/g2/getOnsInfo?name=disease_h5'
other_url = 'https://view.inews.qq.com/g2/getOnsInfo?name=disease_other'
_, china_response = getURLContentGet(china_url)
_, other_response = getURLContentGet(other_url)
china_json_data = json.loads(china_response.text)
other_json_data = json.loads(other_response.text)
china_all_data = json.loads(china_json_data['data'])
other_all_data = json.loads(other_json_data['data'])
china_all_data_area_tree = china_all_data['areaTree'][0] # china_all_data['areaTree']是list对象，只有一个元素
china_all_data_article_list = china_all_data['articleList']
china_all_data_china_day_list = china_all_data['chinaDayList']
china_all_data_china_day_add_list = china_all_data['chinaDayAddList']
china_all_data_daily_new_add_history = china_all_data['dailyNewAddHistory']
china_all_data_daily_history = china_all_data['dailyHistory']
# 下面的后四项和上面的后四项都代表相同的数据，但是，不一定是同时有数据，有一方可能是空列表（目前是上方的后四项为[]） --怀疑原因：2020.3.23-24 在other_all_data有globalDailyHistory元素，但是2020.3.25晚上就找不到该元素了，导致重写程序。
other_all_data_article_list = other_all_data['articleList']
other_all_data_china_day_list = other_all_data['chinaDayList']
other_all_data_china_day_add_list = other_all_data['chinaDayAddList']
other_all_data_daily_new_add_history = other_all_data['dailyNewAddHistory']
other_all_data_daily_history = other_all_data['dailyHistory']
china_day_list = [other_all_data_china_day_list if len(other_all_data_china_day_list) > len(
china_all_data_china_day_list) else china_all_data_china_day_list][0]
daily_new_add_history = \
[other_all_data_daily_new_add_history if len(other_all_data_daily_new_add_history) > len(
china_all_data_daily_new_add_history) else china_all_data_daily_new_add_history][0]
getChinaDataAndSave(workbook, china_day_list, daily_new_add_history, date_list_all, start_date, sheet_name="中国")
if "海外" in choice:
del choice[choice.index("海外")]
foreign_all_data_global_daily_history = foreign_all_data['globalDailyHistory']
foreign_all_data_foreign_list = foreign_all_data['foreignList']
foreign_all_data_global_statis = foreign_all_data['globalStatis']
global_daily_history = foreign_all_data_global_daily_history
getGlobalDataAndSave(workbook, global_daily_history, date_list_all, start_date, sheet_name="海外")
if len(choice) > 0:
getCountryEpidemiSituationInfo(workbook, choice)
# 保存
workbook.save('./疫情原始数据.xls')效果如下：
