链家网爬虫

import requests
from bs4 import BeautifulSoup
headers={
        'UserAgent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36'
}

def get_details(url):
    res=requests.get(url,headers=headers)
    soup=BeautifulSoup(res.text,'html.parser')
    titles=soup.select('#js-ershoufangList > div.content-wrapper > div.content > div.m-list > ul > li > div > div.prop-title > a')
    adresses=soup.select('#js-ershoufangList > div.content-wrapper > div.content > div.m-list > ul > li > div > div.info-table > div > span.info-col.row2-text > a.laisuzhou > span')
    prices=soup.select('#js-ershoufangList > div.content-wrapper > div.content > div.m-list > ul > li > div > div.info-table > div > div > span.total-price.strong-num')
    unit_prices=soup.select('#js-ershoufangList > div.content-wrapper > div.content > div.m-list > ul > li > div > div.info-table > div > span.info-col.price-item.minor')
    dd=[]
    for title,adress,price,unit_price in zip(titles,adresses,prices,unit_prices):
        data={'title':title.get_text().strip(),'adress':adress.get_text().strip(),'price':price.get_text().strip(),'unit_price':unit_price.get_text().strip()}
        dd.append(data)
    return dd


url1='http://sh.lianjia.com/ershoufang/d2'
#print(get_details(url1))


'''
for i in range(1,4):
    urls='http://sh.lianjia.com/ershoufang/d{}'
    urlss=urls.format(i)
    datas=get_details(urlss)
    #print(datas)
'''
lj=[]
for i in  range(1,30):
    urls='http://sh.lianjia.com/ershoufang/' +'d'+str(i)
    #urlss=urls.format(i)
    datas=get_details(urls)
    lj.append(datas)
    #print(datas)

import pandas 
""" 
for p in range(len(lj)):

    lj[p]=pandas.DataFrame(lj[p])
""" 
lj=list(map(pandas.DataFrame,lj))

re= pandas.concat(lj)    

re.index=list(range(re.shape[0]))

re.to_excel('re.xls')

#a1=list(map(pandas.DataFrame,lj))