用python抓取摩拜单车API数据

最近摩拜单车超级火,下班了各种骑,然后膝盖伤了。。。。。。

-----------------------------------------------------------------------

看到这篇文章http://mp.weixin.qq.com/s?__biz=MzA5NDExMTAzNA==&mid=2649982414&idx=1&sn=68b638c4f019baa3a783c045b294d6de&chksm=8854b19bbf23388dd91509a2a692736a5c0505ab144c5605d4015330044699c20e5c42f92496&mpshare=1&scene=1&srcid=0519Fkgd6ew4TuOHUerHMlZs#rd

说好的给源码,但貌似没给。。。。

然后就自己写了一份

--------------------------------------------------------------------------------------

import requests
import numpy as np
import json
import multiprocessing
from requests.packages.urllib3.exceptions import InsecureRequestWarning
#忽略警告
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)

#进程数
PROCESS_NUM = 4
#模拟的数据包头
HEADERS = {
'charset':'utf-8',
'platform':'4',
'referer':'https://servicewechat.com/wx40f112341ae33edb/1/',
'content-type':'application/x-www-form-urlencoded',
'user-agent':'MicroMessenger/6.5.4.1000 NetType/WIFI Language/zh_CN',
'host':'mwx.mobike.com',
'connection':'Keep-Alive',
'accept-encoding':'gzip',
'cache-control':'no-cache'
}
#初始化的部分鹿城区经纬度
LEFT = 28.026735
TOP = 120.644784
RIGHT = 27.99344
BOTTOM = 120.715068
INIT_POS = (TOP, LEFT, BOTTOM, RIGHT)

#将一个区域分成num份,每份后续对应一个进程进行处理
def split_pos(init_pos, num):
result = []
x1 = init_pos[0]
y1 = init_pos[1]
x2 = init_pos[2]
y2 = init_pos[3]
xrange = np.linspace(x1, x2, num + 1)
for i in range(num):
#可以做一些重叠的优化,暂时没做
result.append((xrange[i], y1, xrange[i+1], y2))
return result


#解析并保存数据
#可以直接通过MONGDB操作,这台机器没安装。。
def save(text, output):
try:
json_data = json.loads(text)
except:
print('***ERR1*** ', text)
return
if 'object' not in json_data:
print('***ERR2*** ', text)
return
for object in json_data['object']:
obj_values = []
for key in sorted(object.keys()):
try:
if(object[key] == None):
obj_values.append('')
else:
obj_values.append(str(object[key]))
except:
print('***ERR3*** ', object)
return
output.write('$'.join(obj_values) + '\n')


#爬取摩拜单车的数据,没有对反爬虫的限制进行处理,可以考虑IP池、拨号等手段
#此处只是闹着玩。。
def worker(num, location):
raw_fd = open('worker' + str(num) + '_rawdata.txt', 'w')
tabular_fd = open('worker' + str(num) + '_tabulardata.txt', 'w')
session = requests.session()
url = 'https://mwx.mobike.com/mobike-api/rent/nearbyBikesInfo.do'
# 模拟登录
postdata = {
'latitude': 0,
'longitude': 0,
'errMsg': 'getMapCenterLocation'
}
(top, left, bottom, right) = location
#按照经纬度,一个区域一个区域的获取信息
lat_range = np.arange(left, right, -0.002)
for lat in lat_range:
lon_range = np.arange(top, bottom, 0.002)
for lon in lon_range:
postdata['latitude'] = lat
postdata['longitude'] = lon
mobike_data = session.post(url, headers=HEADERS, data=postdata, verify=False)
#print(mobike_data.text)
#保存原始数据,原始数据还是很关键的,最好都保留下来
raw_fd.write(mobike_data.text + '\n')
#解析并保存数据,还需要去除,后续单独进行,尽量不要再多进程或者多线程环境下处理
save(mobike_data.text, tabular_fd)
raw_fd.close()
tabular_fd.close()

#主函数入口
if __name__ == '__main__':
jobs = []
location = split_pos(INIT_POS, PROCESS_NUM)
#执行多进程,也可以用多线程
for process_id in range(PROCESS_NUM):
p = multiprocessing.Process(target=worker, args=(process_id,location[process_id]))
jobs.append(p)
p.start()

转载于:https://www.cnblogs.com/kernel521/p/6894443.html

你可能感兴趣的:(用python抓取摩拜单车API数据)