Python批量爬取各地方房价走势
文章目录
- 一、需求
- 二、分析
- 三、处理
一、需求
- 批量爬取各城市房价top10涨跌小区
- 记录到.txt中
二、分析
- 网页源代码即可获取数据，通过数据解析方式（Xpath、Bs4、Pyquery即可获取）
三、处理
- Xpath处理
# -*- encoding:utf-8 -*-
__author__ = "nick"
__created_date__ = "2022/10/7"
"""
批量爬取各城市房价走势涨幅top10和跌幅top10
"""
from lxml import etree
import requests
HEADERS = {"user-agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36"}
INDEX = "https://bj.fangjia.com/zoushi"
def process_index(url):
"""
首页处理
@param url: url
@return: 返回首页源代码
"""
res = requests.request("GET", url=url, headers=HEADERS)
res.encoding = "utf-8"
return res.text
def process_city(html):
"""
各城市首页url获取
@param html: 源代码
@return: 返回各城市名和url
"""
parse = etree.HTML(html)
text = parse.xpath('//div[@class="tab_content"]/div')
city_name = []
city_url = []
for i in text:
city_name.extend(i.xpath("./a/text()"))
city_url.extend(i.xpath("./a/@href"))
city_info = dict(zip(city_name,city_url))
return city_info
def process_trend(html):
"""
各城市小区涨跌top处理
@param html: 网页源码
@return: 各城市小区名，房价基本信息
"""
parse = etree.HTML(html)
area = parse.xpath('//div[@class="trend trend03"]/div/div//tbody/tr')
plot_name = []
info = []
for tr in area:
plot_name.extend(tr.xpath("./td/a/text()"))
plot_info = tr.xpath("./td/text()")
base_info = ','.join(plot_info)
info.append(base_info)
plot_intend = dict(zip(plot_name,info))
return plot_intend
if __name__ == '__main__':
index_html = process_index(INDEX)
city_dict = process_city(index_html)
# 记录到文件中
f = open('全国各城市房价小区涨跌top10_by_xpath.txt',"w", encoding="utf-8")
# 批量获取各城市房价涨跌幅top10
for city_name, city_url in city_dict.items():
# 城市首页处理
city_html = process_index(city_url)
# 房价涨跌top10
plot_intend = process_trend(city_html)
if bool(plot_intend):
for k,value in plot_intend.items():
f.write(f"城市{city_name}----小区名--{k}---房价基本信息{value}\n")
print(f"城市{city_name}----小区名{k}下载完毕....")
else:
f.write(f"城市{city_name}无涨幅小区top10\n")
f.write(f"-------------------城市{city_name}分隔线--------------------------\n")
f.close()
2、Pyquery 处理
# -*- encoding:utf-8 -*-
__author__ = "nick"
__created_date__ = "2022/10/7"
"""
批量爬取各城市房价走势涨幅top10和跌幅top10
"""
from pyquery import PyQuery as pq
import requests
HEADERS = {"user-agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36"}
INDEX = "https://bj.fangjia.com/zoushi"
def process_index(url):
"""
首页处理
@param url: url
@return: 返回首页源代码
"""
res = requests.request("GET", url=url, headers=HEADERS)
res.encoding = "utf-8"
return res.text
def process_city(html):
"""
各城市首页url获取
@param html: 源代码
@return: 返回各城市名和url
"""
doc = pq(html)
tab = doc(".tab_content")
a = tab("a")
city_name = []
city_url = []
for info in a.items():
city_name.append(info.text())
city_url.append(info.attr("href"))
city_info = dict(zip(city_name,city_url))
return city_info
def process_trend(html="https://ak.fangjia.com/zoushi"):
"""
各城市小区涨跌top处理
@param html: 网页源码
@return: 各城市小区名，房价基本信息
"""
doc = pq(html)
tab = doc(".trend03")
plot_name = []
info = []
for tr in tab.items():
for n in tr("tr a").items():
name = n.text()
plot_name.append(name)
for td in tr("tr").items():
house_info = td("td").text()
house_info.strip("\n")
info.append(house_info)
plot_intend = dict(zip(plot_name,info))
return plot_intend
if __name__ == '__main__':
index_html = process_index(INDEX)
city_dict = process_city(index_html)
# 记录到文件中
f = open('全国各城市房价小区涨跌top10_by_pyquery.txt',"w", encoding="utf-8")
# 批量获取各城市房价涨跌幅top10
for city_name, city_url in city_dict.items():
# 城市首页处理
city_html = process_index(city_url)
# 房价涨跌top10
plot_intend = process_trend(city_html)
if bool(plot_intend):
for k,value in plot_intend.items():
f.write(f"城市{city_name}----小区名--{k}---房价基本信息{value}\n")
print(f"城市{city_name}----小区名{k}下载完毕....")
else:
f.write(f"城市{city_name}无涨幅小区top10\n")
f.write(f"-------------------城市{city_name}分隔线--------------------------\n")
f.close()