爬虫实例5（爬取房天下二手房信息）（网页重定向，字典数据存入csv文件，bs4）

爬取要点分析

***声明：***纯粹是用于学习
1.网页重定向
分析房天下网站，每个网页有个重定向
如：访问https://cd.esf.fang.com/chushou/3_211293494.htm会跳转至https://cd.esf.fang.com/chushou/3_211293494.htm?rfss=1-b71f212cbb874a451c-3a
解决方法：在原网页源代码中找到重定向网址，request 新网址即可

 response=requests.get(url,headers = headers)
    html=response.text
    #网页重定向
    pat=re.compile(r'点击跳转')
    url=re.findall(pat,html)[0]

    response=requests.get(url,headers = headers)
    return response.text

2.bs4获取标签内容，部分代码

temp_dict['房源']=soup.find('title').string
temp_dict['小区'] = soup.find('div',id="xq_message").get_text()
temp_dict['总价']=soup.find('div',class_="tab-cont-right").find('div',class_="trl-item price_esf sty1").get_text()

3.将数据保存在csv文件中

def save_data_csv(keyword_list,dict_data):

    if not os.path.exists('fang.csv'):
        with open('fang.csv', "w", newline='', encoding='utf-8') as csvfile:  # newline='' 去除空白行
            writer = csv.DictWriter(csvfile, fieldnames=keyword_list)  # 写字典的方法
            writer.writeheader()  # 写表头的方法

        # 接下来追加写入内容
    with open('fang.csv', "a", newline='', encoding='utf-8') as csvfile:  # newline='' 一定要写，否则写入数据有空白行
            writer = csv.DictWriter(csvfile, fieldnames=keyword_list)
            writer.writerow(dict_data)  # 按行写入数据
            print("^_^ write success")

4.完整代码

import re,requests,time,os
from bs4 import BeautifulSoup
from lxml import etree
import json
import csv
def get_html(url):
    headers={
        'user-agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.92 Safari/537.36',
        'cookie':'city=cd; __utma=147393320.989730142.1589024416.1589024416.1589024416.1; __utmc=147393320; __utmz=147393320.1589024416.1.1.utmcsr=baidu|utmccn=(organic)|utmcmd=organic; __utmt_t0=1; __utmt_t1=1; __utmt_t2=1; global_cookie=9ekumblkqetgf7unt5iefiegk1nk9zk41by; logGuid=08b2183e-66fd-4851-8c49-20b9c55f4562; Integrateactivity=notincludemc; csrfToken=ZEhmg2XlXN8rivcJcEqBk4FO; budgetLayer=1%7Ccd%7C2020-05-09%2019%3A41%3A50; g_sourcepage=esf_fy%5Exq_pc; lastscanpage=0; __utmb=147393320.15.10.1589024416; unique_cookie=U_9ekumblkqetgf7unt5iefiegk1nk9zk41by*4',
        'authority': 'cd.esf.fang.com',
        'path': '/staticsearchlist/EsfListAjax/GetAIDaoGou?pagesize=5',
    }
    response=requests.get(url,headers = headers)
    html=response.text
    #网页重定向
    pat=re.compile(r'点击跳转')
    url=re.findall(pat,html)[0]
    response=requests.get(url,headers = headers)
    return response.text
    #print(html)
def get_fang_url(html):
    soup = etree.HTML(html)
    urls = soup.xpath('//dd/h4[@class="clearfix"]/a/@href')
    return urls
def save_data_csv(keyword_list,dict_data):

    if not os.path.exists('fang.csv'):
        with open('fang.csv', "w", newline='', encoding='utf-8') as csvfile:  # newline='' 去除空白行
            writer = csv.DictWriter(csvfile, fieldnames=keyword_list)  # 写字典的方法
            writer.writeheader()  # 写表头的方法

        # 接下来追加写入内容
    with open('fang.csv', "a", newline='', encoding='utf-8') as csvfile:  # newline='' 一定要写，否则写入数据有空白行
            writer = csv.DictWriter(csvfile, fieldnames=keyword_list)
            writer.writerow(dict_data)  # 按行写入数据
            print("^_^ write success")
def parse_page(url,html):
    #去除网页html里面的换行，以便更好的获取数据
    html = "".join(line.strip() for line in html.split("\n"))
    #定义一个字典
    temp_dict={}

    soup=BeautifulSoup(html,'lxml')
    #采集房源信息
    temp_dict['房源']=soup.find('title').string
    temp_dict['小区'] = soup.find('div',id="xq_message").get_text()

    temp_dict['总价']=soup.find('div',class_="tab-cont-right").find('div',class_="trl-item price_esf sty1").get_text()
    temp_dict['户型']=soup.find('div',class_="trl-item1 w146").find('div',class_="tt").get_text()
    temp_dict['建筑面积'] = soup.find('div', class_="trl-item1 w182").find('div', class_="tt").get_text()
    temp_dict['单价'] = soup.find('div', class_="trl-item1 w132").find('div', class_="tt").get_text()
    temp_dict['详情页'] = url
    temp_dict['经纪人'] = soup.find('a',id="kesfsfbxq_A01_03_03").get_text()
    print(temp_dict)
    keyword_list=['房源','小区','总价','户型','建筑面积','单价','详情页','经纪人']
    save_data_csv(keyword_list,temp_dict)
#主函数
if __name__ == '__main__':
    #构造网址序列
    urls=['https://cd.esf.fang.com/house-a016749-b014906/i{}'.format(i) for i in range(1,101)]
    for url in urls:
        #获取单页数据
        html = get_html(url)
        #获取单页上房源的链接
        new_urls = get_fang_url(html)
        for url in new_urls:
            new_url = 'https://cd.esf.fang.com' + url
        #打印网址
            print(new_url)
        #获取HTML
            html = get_html(new_url)
        #解析网页，保存数据
            parse_page(new_url,html)
    #防止被禁IP，每访问完一页，睡眠5秒
    time.sleep(5)