frombs4importBeautifulSoup
importrequests
importtime
url=['http://bj.xiaozhu.com/search-duanzufang-p{}-0/?startDate=2016-06-13&endDate=2016-06-14'.format(i)foriin
range(1,15,1)]
lianjie1=[]
deflian(url1):
info=[]
wb_data=requests.get(url1)
soup=BeautifulSoup(wb_data.text,'lxml')
time.sleep(2)
lianjie=soup.find_all(style='cursor:pointer')
foriinlianjie:
abc=i.get('detailurl')
url=abc
wb_data=requests.get(url)
soup=BeautifulSoup(wb_data.text,'lxml')
titles=soup.select(' h4 > em')
addresss=soup.select('p > span.pr5')
prices=soup.select('div.day_l > span')
images=soup.find_all(id='curBigImage')
imagespeople=soup.select('div.member_pic > a > img')
sexs=soup.select('div.member_pic > div')
name_oweners=soup.select('div.w_240 > h6 > a')
defsexss(valuse):
job=valuse
job3=[]
foriinjob:
job1=i.get('class')
ifjob1[0]=='member_ico1':
job2='女'
job3.append(job2)
elifjob1[0]=='memeber_ico':
job2='男'
job3.append(job2)
else:
job2='性别未知'
job3.append(job2)
return(job3)
job4=sexss(sexs)
fortitle,address,price,image,imagepeople,sex,name_owenerinzip(titles,addresss,prices,images,imagespeople,sexs,name_oweners):
data={
'title':title.get_text(),
'address':address.get_text(),
'price':price.get_text(),
'image':image.get('src'),
'imagepeople':imagepeople.get('src'),
'sex':job4,
'name_owener':name_owener.get_text()
}
info.append(data)
foriininfo:
print(i['title'], i['address'],str( i['price'])+'￥', i['image'], i['imagepeople'],i['sex'],i['name_owener'])
foriinurl:
countent=lian(i)
这个项目做完后，我进一步对网页的构成加深了理解，并建立了玩虫的决心，但是发现，还是存在老问题，就是爬出来的数据有大量重复的，希望老师能给指导，下一步的研究方向是python对execl的操作！