基于高德api的地区全类poi爬取
最近花了大概两个月时间学习python以及爬虫，初见成效，完成了一个基于高德api的全类poi爬取小项目，特此分享。关于高德api的使用可以在此查看：搜索POI 2.0-API文档-开发指南-Web服务 API | 高德地图API (amap.com)
(关于我的各项方法以及py文件命名方面可能不是很规范，欢迎大家批评指正)
主方法（获取数据，插入数据库，查询数据是否已爬取（方便随时停止随时开始））：
from db_base import get_sub_cat
from get_data import get_data
from pgsql_insert import handle_data, insert_list
from get_adcode import get_adcode
from get_list_from_pg import get_list
def main_poi():
table_name = '**_**' # 你需要插入数据的表
list_adcode = get_adcode() # 获取需要爬取的区域的
for i in range(len(list_adcode)):
adcode = list_adcode[i]
f = "adcode"
tn = "base.adcode_for_get"
list_judge = get_list(f, tn)
if adcode in list_judge: # 判断区域是否已爬取
print("该区已爬取，跳过")
else:
list_num_cat = ['01', '02', '03', '04', '05', '06', '07', '08', '09',
'10', '11', '12', '13', '14', '15', '16', '17', '18']
# 高德poi分类统共18个大类，用于生成请求条件
for a in range(len(list_num_cat)):
list_types_test = get_sub_cat(list_num_cat[a])
types = ''
for b in range(len(list_types_test)):
types = types + list_types_test[b] + '|'
code_adcode = types + adcode
f_ca = "code_adcode"
tn_ca = "base.code_adcode_for_get"
list_judge_ca = get_list(f_ca, tn_ca)
if code_adcode in list_judge_ca: # 判断某区某类是否已爬取
print("该条数据已爬取，跳过")
else:
list_poi = get_data(types, adcode)
handle_data(table_name, list_poi) # 插入poi数据方法
data = types + adcode
tn1 = 'code_adcode_for_get'
tf = 'code_adcode'
insert_list(tn1, tf, data) # 插入types+adcode数据用来记录该数据已爬取
tf1 = 'adcode'
insert_list(tn, tf1, adcode) # 插入adcode数据记录该区域已爬取
print("爬取完成")获取网页数据方法：
import requests
from db_base import get_sub_cat
def get_data(types, adcode):
list_result_poi = [] # 最终返回值作为插入数据库的数据
list_amap_key = [] # 可去高德申请key，建议多申请几个，一个号一天可以请求5000次
i = 0
key = list_amap_key[i]
count = '25' # 由于高德每次请求每页最多给25条poi信息，故利用其进行实现分页的查询
for j in range(1, 100):
if count == '25':
page_num = j
url = " https://restapi.amap.com/v5/place/text?" \
"key={}&types={}®ion={}&page_num={}" \
"&citylimit=true&extensions=all&show_fields=navi,business&page_size=25" \
.format(key, types, adcode, page_num)
print(url)
r = requests.get(url)
js = r.json() # 将网页数据转为json
status = js['status']
if status == '1': # 判断请求是否成功
count = js['count']
re = js['pois']
list_result_poi.append(re)
elif status == '0': # 请求失败则换key
i = i + 1
key = list_amap_key[i]
return list_result_poi # 最后以一个列表字典的形式返回获取adcode（关于adcode可见相关下载-Web服务 API | 高德地图API (amap.com)）：
import psycopg2
def get_adcode(): # 可根据你输入的是省还是市来获取下级区域的adcode
conn = psycopg2.connect() # 数据库信息
city = input("请输入你要查询的城市(请输入省或者市并在后面加上省或市):")
curs = conn.cursor()
select_sql1 = "select adcode from base.base_poi_city where name = " + "'" + city + "'"
curs.execute(select_sql1)
adcode1 = curs.fetchall()[0][0]
if adcode1[2:6] != '0000':
adcode2 = str(int(int(adcode1)/100))
elif adcode1[2:6] == '0000':
adcode2 = str(int(int(adcode1)/1000))
curs.close()
curs = conn.cursor()
select_sql2 = "select adcode from base.base_poi_city where adcode like " + "'" + adcode2 + "%'"
curs.execute(select_sql2)
data = curs.fetchall()
curs.close()
conn.close()
list_sub_poi = []
for i in range(len(data)):
re = data[i][0]
if re[4:6] != '01' and re[4:6] != '00':
list_sub_poi.append(re)
return list_sub_poi
获取poi分类（关于poi分类可见上面链接）：
import psycopg2
def get_sub_cat(num_cat):
conn = psycopg2.connect() # 数据库信息
curs = conn.cursor()
select_sql1 = "select code from base.base_poi_cat where code like'" + num_cat + "%'"
# SQL语句选取行业分类编号code
curs.execute(select_sql1)
data = curs.fetchall()
curs.close()
conn.close()
list_sub_poi = []
for a in range(len(data)):
re = data[a][0]
list_sub_poi.append(re) # 将获取到的poi编码生成为一个列
return list_sub_poi
插入数据库方法：
import traceback
from src.utils import common
import psycopg2
import math
from src.utils import zuobiaozhuanhuan as zbzh
def handle_data(table_name, list_result0): # 插入数据的主要方法，根据自己的需求改
conn = psycopg2.connect()
cursor = conn.cursor()
dict_poi = {
"id": "",
"大类": "",
"中类": "",
"小类": "",
"兴趣点名称": "",
"百度坐标": "",
"省名称": "",
"省代号": "",
"市名称": "",
"市代号": "",
"区县名称": "",
"区县代号": "",
"地址": "",
"地理格ID": "",
"类型代号": "",
"电话": "",
"84坐标": "",
"更新时间": "",
} # 作为数据缓存字典。然后转入数据库中
for i in range(len(list_result0)):
list_result1 = list_result0[i]
for i in range(len(list_result1)):
list_result = list_result1[i]
"""
将高德坐标解密为xy84
再将xy84转化为百度坐标
"""
x_pi = 3.14159265358979324 * 3000.0 / 180.0
x = str(list_result['location']).split(',')[0]
y = str(list_result['location']).split(',')[1]
[x0, y0] = zbzh.gcj02_to_wgs84(float(x), float(y)) # 高德加密经纬度（火星）转WGS84方法
z = math.sqrt(float(x0) * float(x0) + float(y0) * float(y0)) + 0.00002 * math.sin(float(y0) * x_pi)
theta = math.atan2(float(y0), float(x0)) + 0.000003 * math.cos(float(x0) * x_pi)
lng = z * math.cos(theta) + 0.0065
lat = z * math.sin(theta) + 0.006
xy84 = str(x0) + ',' + str(y0)
xybaidu = str(lng) + ',' + str(lat)
type1 = str(list_result['type'])
dict_poi['id'] = list_result['id']
dict_poi['大类'] = type1.split(';')[0]
dict_poi['中类'] = type1.split(';')[1]
dict_poi['小类'] = type1.split(';')[2]
dict_poi['兴趣点名称'] = list_result['name']
dict_poi['百度坐标'] = xybaidu
dict_poi['省名称'] = list_result['pname']
dict_poi['省代号'] = list_result['pcode']
dict_poi['市名称'] = list_result['cityname']
dict_poi['市代号'] = list_result['citycode']
dict_poi['区县名称'] = list_result['adname']
dict_poi['区县代号'] = list_result['adcode']
dict_poi['地址'] = list_result['address']
dict_poi['84坐标'] = xy84
dict_poi['类型代号'] = list_result['typecode']
try:
dict_poi['地理格ID'] = list_result['navi']['gridcode']
except Exception:
print('该条无地理格信息')
try:
dict_poi['电话'] = list_result['business']['tel']
except Exception:
print('该条无电话信息')
try:
update_time = common.get_now()
sql = (
"insert into "
+ table_name
+ "(ID,BIG_TYPE,MOD_TYPE,SMALL_TYPE,NAME,XY,PNAME,PCODE,CITYNAME,CITYCODE,ADNAME,ADCODE,UPDATE_TIME,ADDRESS,"
+ "GRIDCODE,TYPECODE,TEL,XY84)"
+ "values("
+ "'"
+ str(dict_poi["id"])
+ "','"
+ str(dict_poi["大类"])
+ "','"
+ str(dict_poi["中类"])
+ "','"
+ str(dict_poi["小类"])
+ "','"
+ str(dict_poi["兴趣点名称"])
+ "','"
+ str(dict_poi["百度坐标"])
+ "','"
+ str(dict_poi["省名称"])
+ "','"
+ str(dict_poi["省代号"])
+ "','"
+ str(dict_poi["市名称"])
+ "','"
+ str(dict_poi["市代号"])
+ "','"
+ str(dict_poi["区县名称"])
+ "','"
+ str(dict_poi["区县代号"])
+ "','"
+ update_time
+ "','"
+ str(dict_poi["地址"])
+ "','"
+ str(dict_poi["地理格ID"])
+ "','"
+ str(dict_poi['类型代号'])
+ "','"
+ str(dict_poi['电话'])
+ "','"
+ str(dict_poi['84坐标'])
+ "')"
)
print(sql)
cursor.execute(sql)
conn.commit()
except Exception:
print("保存数据出错: {}".format(traceback.format_exc()))
conn.commit()
print('数据导入成功')
cursor.close()
def insert_list(table_name, table_feild, data): # 主要用于对已爬取数据记录的插入
conn = psycopg2.connect()
cursor = conn.cursor()
sql = (f"""insert into {table_name} ({table_feild}) values ('{data}')""")
cursor.execute(sql)
conn.commit()
cursor.close()
获取数据展示：
