本文是利用Python对链家网长沙房源信息进行数据采集、提取并进行分析的练习。
先总结下：
- user—agent ： 采用万行的user列表，每次随机使用，伪造浏览器以及屏幕和系统等信息；
- cookie ：带真实cookie；
- 随机休息时间；
- 使用logging库和相关函数提醒进度和错误，方便查找错误；
- 尝试采用peewee库和数据库函数将数据导入mysql中。
数据采集
经过对目标网站的网页进行分析可以知道，目标数据都存在源代码中。
浏览器伪装并爬取网页
import requests
import random
from datetime import datetime
from bs4 import BeautifulSoup
import logging
import time
from peewee import *
hds = [{'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'},
{'User-Agent': 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.12 Safari/535.11'},
{'User-Agent': 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0)'},
{'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:34.0) Gecko/20100101 Firefox/34.0'},
{'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/44.0.2403.89 Chrome/44.0.2403.89 Safari/537.36'},
{'User-Agent': 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50'},
{'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50'},
{'User-Agent': 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0'},
{'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1'},
{'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1'},
{'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11'},
{'User-Agent': 'Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11'},
{'User-Agent': 'Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11'}]
hd = {
'Host': 'cs.lianjia.com',
'Connection': 'keep-alive',
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Referer': 'https://cs.lianjia.com/zufang/yuhua/',
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Language': 'zh-CN,zh;q=0.9',
'Cookie': 'lianjia_uuid=96b42f77-56b8-490d-baa9-beaf6865a9c4; _smt_uid=5c724732.b80d85e; UM_distinctid=1691e661dd36c9-08249c43205ce8-3d644701-144000-1691e661dd44ff; _jzqc=1; _ga=GA1.2.186869930.1550993207; _gid=GA1.2.1359688634.1550993207; select_city=430100; _jzqx=1.1550993203.1551081745.3.jzqsr=cn%2Ebing%2Ecom|jzqct=/.jzqsr=cs%2Elianjia%2Ecom|jzqct=/xiaoqu/wangcheng/; _jzqckmp=1; Hm_lvt_9152f8221cb6243a53c83b956842be8a=1550993227,1550993401,1551082378,1551095350; _jzqa=1.3371712841637333500.1550993203.1551094837.1551107638.6; Hm_lpvt_9152f8221cb6243a53c83b956842be8a=1551108287; lianjia_ssid=ac48bf96-5661-bd1a-2293-92ba40145fa0; CNZZDATA1273627291=1275184754-1551151174-https%253A%252F%252Fcs.lianjia.com%252F%7C1551151174'
}
logging.basicConfig(
format='%(asctime)s - %(levelname)s - %(message)s', level=logging.INFO)
#设定日志提醒格式和等级
def get_source_code(url):
try:
result = requests.get(
url, headers=hds[random.randint(0, len(hds) - 1)])
source_code = result.content
except Exception as e:
print(e)
return
return source_code
#随机使用user_agent
提取网页信息
经过分析网页逻辑，发现通过分地区然后分小区获取房源比较全面
该部分函数逻辑较复杂 先进行简单介绍：
行政区列表：regionlist ；
小区列表：通过get_community函数可以得到；
小区信息：getCommunity_byRegionlist（regionlist）：该函数根据行政区返回小区信息；
日志控制：check_blocked函数用于及时提醒因为爬取速度过快导致需要输入验证码的情况；log_progress：实时提供爬虫进度信息。
获取每个地区小区页面数量
regionlist = ['yuhua','yuelu','tianxin','kaifu','furong','wangcheng']
# 地区列表
def get_total_pages(url):
source_code = get_source_code(url)
soup = BeautifulSoup(source_code, 'lxml')
total_pages = 0
try:
page_info = soup.find('div', {'class': 'page-box house-lst-page-box'})
except AttributeError as e:
page_info = None
if page_info == None:
logging.error('can not find pages')
return
page_info_str = page_info.get('page-data').split(',')[0]
total_pages = int(page_info_str.split(':')[1])
return total_pages
获取每个地区的小区信息
def get_community(region):
baseUrl = 'https://cs.lianjia.com/xiaoqu/'
url = baseUrl + region+'/'
source_code = get_source_code(url)
soup = BeautifulSoup(source_code, 'lxml')
if check_blocked(soup):
return
total_pages = get_total_pages(url)
for page in range(total_pages):
if page > 0:
url_page = baseUrl + region + "/pg%d/" % page
source_code = get_source_code(url_page)
soup = BeautifulSoup(source_code, 'lxml')
nameList = soup.find_all("li", {"class": "clear"})
i = 0
log_progress("GetCommunity",
region, page + 1, total_pages)
for name in nameList:
info_dict = {}
communitytitle = name.find("div", {"class": "title"})
title = communitytitle.get_text().strip('\n')
link = communitytitle.a.get('href')
info_dict.update({u'title': title})
info_dict.update({u'link': link})
district = name.find("a", {"class": "district"})
info_dict.update({u'district': district.get_text()})
bizcircle = name.find("a", {"class": "bizcircle"})
info_dict.update({u'bizcircle': bizcircle.get_text()})
tagList = name.find("div", {"class": "tagList"})
info_dict.update({u'tagList': tagList.get_text().strip('\n')})
onsale = name.find("a", {"class": "totalSellCount"})
info_dict.update(
{u'onsale': onsale.span.get_text().strip('\n')})
onrent = name.find("a", {"title": title + u"租房"})
info_dict.update(
{u'onrent': onrent.get_text().strip('\n').split(u'套')[0]})
info_dict.update({u'id': name.get('data-housecode')})
price = name.find("div", {"class": "totalPrice"})
info_dict.update({u'price': price.span.get_text().strip('\n')})
communityinfo = get_communityinfo_by_url(link)
for key, value in communityinfo.items():
info_dict.update({key: value})
#连接数据库
Community.insert(info_dict).on_conflict('replace').execute()
time.sleep(random.randint(1,5)) #时间延迟
def get_communityinfo_by_url(url):
source_code = get_source_code(url)
soup = BeautifulSoup(source_code, 'lxml')
if check_blocked(soup):
return
communityinfos = soup.find_all("div", {"class": "xiaoquInfoItem"})
res = {}
for info in communityinfos:
key_type = {
"建筑年代": 'year',
"建筑类型": 'housetype',
"物业费用": 'cost',
"物业公司": 'service',
"开发商": 'company',
"楼栋总数": 'building_num',
"房屋总数": 'house_num',
}
try:
key = info.find("span", {"xiaoquInfoLabel"})
value = info.find("span", {"xiaoquInfoContent"})
key_info = key_type[key.get_text().strip()]
value_info = value.get_text().strip()
res.update({key_info: value_info})
except:
continue
return res
def getCommunity_byRegionlist(regionlist):
logging.info("Get Community Infomation")
starttime = datetime.now()
for region in regionlist:
try:
get_community(region)
logging.info(region + "Done")
except Exception as e:
logging.error(e)
logging.error(region + "Fail")
pass
endtime = datetime.now()
logging.info("Run time: " + str(endtime - starttime))
日志控制
def check_blocked(soup):
if soup.title.string == '414 Request-URI Too Large':
logging.error('IP is blocked')
return True
return False
def log_progress(function, address, page, total):
logging.info("Progress: %s %s: current page %d total pages %d" %
(function, address, page, total))
数据库设置
database = MySQLDatabase(
'house',
host='localhost',
port=3306,
user='root',
passwd='',
charset='utf8',
use_unicode=True,
)
class BaseModel(Model):
class Meta:
database = database
class Community(BaseModel):
id = BigIntegerField(primary_key=True)
title = CharField()
link = CharField(unique=True)
district = CharField()
bizcircle = CharField()
tagList = CharField(null=True)
onsale = CharField()
onrent = CharField(null=True)
year = CharField(null=True)
housetype = CharField(null=True)
cost = CharField(null=True)
service = CharField(null=True)
company = CharField(null=True)
building_num = CharField(null=True)
house_num = CharField(null=True)
price = CharField(null=True)
validdate = DateTimeField(default=datetime.now)
def database_init():
database.connect()
database.create_tables(Community, safe=True)
database.close()
#数据库表设定