汽车之家爬虫实例

import json
import time
import urllib
import urllib2
import re



#获取城市列表
url='https://dealer.autohome.com.cn/DealerList/GetAreasAjax?provinceId=340000&cityId=0&brandid=0&manufactoryid=0&seriesid=0&isSales=0'
html=urllib2.urlopen(url)
html=html.read()
value=json.loads(html.decode('gbk'))
lis=[]
r=len(value['AreaInfoGroups'])
for i in range(0,r):
    len1=len(value['AreaInfoGroups'][i]['Values'])
    for j in range(0,len1):
        len2=len(value['AreaInfoGroups'][i]['Values'][j]['Cities'])
        for z in range(0,len2):
            lis.append( str(value['AreaInfoGroups'][i]['Values'][j]['Cities'][z]['Pinyin'])) 

print lis


















        

-----------------------------------------------------------------------------------------------------------------------------------------------------------------------

 

#coding:utf-8
import json
import time
import urllib
import urllib2
import re

h=open('qichezhijia01.txt','a')
def run(city,num):
    url='https://dealer.autohome.com.cn/'+str(city)+'/0/0/0/0/'+str(num)+'/1/0/0.html'
    html=urllib2.urlopen(url)
    html=html.read()
    
    ss=get(city,html)
    if ss=='q':
        return 'q'

def get(city,html):
    #print html
    reg4='(.*?).*?' #地址
    reg1='.*?(.*?).*?' #名称
    reg3='(.*?).*?'#电话
    reg2='(.*?).*?' #主营品牌
    reg=reg1+reg2+reg3+reg4
    reg=reg.decode('utf-8')
    reg = re.compile(ur'%s' % reg,re.S)
    items=re.findall(reg,html)
    if not items:
        return 'q'
    for i in  items:
        ss=city+'///'
        for v in i:
            ss+=v
            ss=ss+'///'
        h.write( ss+'\n'  )
    h.flush()
    time.sleep(0.1)




lis=['beijing', 'tianjin', 'shanghai', 'chongqing', 'hefei', 'wuhu', 'bangbu', 'huainan', 'maanshan', 'huaibei',
     'tongling', 'anqing', 'huangshan', 'chuzhou', 'fu_yang', 'su_zhou', 'liuan', 'bozhou', 'chizhou', 'xuancheng',
     'aomen', 'fuzhou', 'xiamen', 'putian', 'sanming', 'quanzhou', 'zhangzhou', 'nanping', 'longyan', 'ningde',
     'guangzhou', 'shaoguan', 'shenzhen', 'zhuhai', 'shantou', 'foshan', 'jiangmen', 'zhanjiang', 'maoming',
     'zhaoqing', 'huizhou', 'meizhou', 'shanwei', 'heyuan', 'yangjiang', 'qingyuan', 'dongguan', 'zhongshan',
     'chaozhou', 'jieyang', 'yunfu', 'nanning', 'liuzhou', 'guilin', 'wuzhou', 'beihai', 'fangchenggang', 'qinzhou',
     'guigang', 'yu_lin', 'baise', 'hezhou', 'hechi', 'laibin', 'chongzuo', 'guiyang', 'liupanshui', 'zunyi',
     'anshun', 'bijie', 'tongren', 'xingyishi', 'kaili', 'duyunshi', 'lanzhou', 'jiayuguan', 'jinchang', 'baiyin',
     'tianshui', 'wuwei', 'zhangye', 'pingliang', 'jiuquan', 'qingyang', 'dingxi', 'longnan', 'linxia', 'gannan',
     'shijiazhuang', 'tangshan', 'qinhuangdao', 'handan', 'xingtai', 'baoding', 'zhangjiakou', 'chengde', 'cangzhou',
     'langfang', 'hengshui', 'haerbin', 'qiqihaer', 'jixi', 'hegang', 'shuangyashan', 'daqing', 'yichun', 'jiamusi',
     'qitaihe', 'mudanjiang', 'heihe', 'suihua', 'daxinganling', 'zhengzhou', 'kaifeng', 'luoyang', 'pingdingshan',
     'anyang', 'hebi', 'xinxiang', 'jiaozuo', 'puyang', 'xuchang', 'luohe', 'sanmenxia', 'nanyang', 'shangqiu',
     'xinyang', 'zhoukou', 'zhumadian', 'jiyuan', 'wuhan', 'huangshi', 'shiyan', 'yichang', 'xiangyang', 'ezhou',
     'jingmen', 'xiaogan', 'jingzhou', 'huanggang', 'xianning', 'suizhou', 'enshi', 'xiantao', 'qianjiang', 'tianmen',
     'shennongjia', 'changsha', 'zhuzhou', 'xiangtan', 'hengyang', 'shaoyang', 'yueyang', 'changde', 'zhangjiajie', 'yiyang',
     'chenzhou', 'yongzhou', 'huaihua', 'loudi', 'xiangxi', 'haikou', 'sanya', 'sanshashi', 'danzhou', 'wuzhishan', 'qionghai',
     'wenchang', 'wanning', 'dongfang', 'dingan', 'tunchang', 'chengmai', 'lingao', 'baisha', 'changjiang', 'ledong',
     'lingshui', 'baoting', 'qiongzhong', 'changchun', 'jilinshi', 'siping', 'liaoyuan', 'tonghua', 'baishan', 'songyuan',
     'baicheng', 'yanbian', 'nanjing', 'wuxi', 'xuzhou', 'changzhou', 'suzhou', 'nantong', 'lianyungang', 'huaian', 'yancheng',
     'yangzhou', 'zhenjiang', 'tai_zhou', 'suqian', 'nanchang', 'jingdezhen', 'ping_xiang', 'jiujiang', 'xinyu', 'yingtan',
     'ganzhou', 'jian', 'yi_chun', 'fu_zhou', 'shangrao', 'shenyang', 'dalian', 'anshan', 'fushun', 'benxi', 'dandong',
     'jinzhou', 'yingkou', 'fuxin', 'liaoyang', 'panjin', 'tieling', 'chaoyang', 'huludao', 'huhehaote', 'baotou', 'wuhai',
     'chifeng', 'tongliao', 'eerduosi', 'hulunbeier', 'bayannaoer', 'wulanchabu', 'xinganmeng', 'xilinguolemeng', 'alashanmeng',
     'yinchuan', 'shizuishan', 'wuzhong', 'guyuan', 'zhongwei', 'xining', 'haidong', 'haibei', 'huangnan', 'hai_nan',
     'guoluo', 'yushu', 'haixi', 'taiyuan', 'datong', 'yangquan', 'changzhi', 'jincheng', 'shuozhou', 'jinzhong', 'yuncheng',
     'xinzhou', 'linfen', 'lvliang', 'jinan', 'qingdao', 'zibo', 'zaozhuang', 'dongying', 'yantai', 'weifang', 'jining', 'taian',
     'weihai', 'rizhao', 'laiwu', 'linyi', 'dezhou', 'liaocheng', 'binzhou', 'heze', 'chengdu', 'zigong', 'panzhihua', 'luzhou',
     'deyang', 'mianyang', 'guangyuan', 'suining', 'neijiang', 'leshan', 'nanchong', 'meishan', 'yibin', 'guangan', 'dazhou',
     'yaan', 'bazhong', 'ziyang', 'aba', 'ganzi', 'liangshan', 'xian', 'tongchuan', 'baoji', 'xianyang', 'weinan', 'yanan',
     'hanzhong', 'yulin', 'ankang', 'shangluo', 'taiwan', 'lasa', 'rikaze', 'changdou', 'linzhi', 'shannan', 'naqu', 'ali',
     'wulumuqi', 'kelamayi', 'tulufan', 'hami', 'changji', 'boertala', 'bayinguoleng', 'akesu', 'kezilesu', 'kashen', 'hetian',
     'yili', 'tacheng', 'aletai', 'shihezi', 'alaer', 'tumushuke', 'wujiaqu', 'beitunshi', 'tiemenguanshi', 'shuangheshi',
     'kekedalashi', 'kunyu', 'xianggang', 'kunming', 'qujing', 'yuxi', 'baoshan', 'zhaotong', 'lijiang', 'puer', 'lincang',
     'chuxiong', 'honghe', 'wenshan', 'xishuangbanna', 'dali', 'dehong', 'nujiang', 'diqing', 'hangzhou', 'ningbo', 'wenzhou',
     'jiaxing', 'huzhou', 'shaoxing', 'jinhua', 'quzhou', 'zhoushan', 'taizhou', 'lishui']

for i in lis:
    print i
    j=1
    while 1:        
        try:
            ss=run(i,j)
            if ss=='q':
                break
            j+=1
            print j
        except:
            break