北京安居客二手房信息爬取

import requests
from lxml import etree
#import time
from selenium import webdriver
import pandas as pd
from bs4 import BeautifulSoup

#需要将chromedriver放到Chrome\Application目录下
#chrome_driver=r"C:\Program Files (x86)\Google\Chrome\Application\chromedriver_win32\chromedriver.exe"
chrome_driver=“C:\Program Files (x86)\Google\Chrome\Application\chromedriver_win32\chromedriver.exe”

driver = webdriver.Chrome(executable_path=chrome_driver)
request_url = ‘https://beijing.anjuke.com/sale/p’

print(html)
html = etree.HTML(html)
#print(html)
#去空格，去换行\n
def format_str(str):
return str.replace(’\n’, ‘’).replace(’ ', ‘’)

houses = pd.DataFrame(columns=[“name”, “details”, “address”, “price”])
for i in range(3):
url = request_url + str(i+1)

driver.get(url)

html = driver.find_element_by_xpath("//*").get_attribute("outerHTML")


soup = BeautifulSoup(html, "html.parser", from_encoding="utf-8")
house_list = soup.find_all("li", class_="list-item")

for house in house_list:
    temp = {}
 #提取房源信息
    name = house.find("div", class_="house-title").a.text.strip()
    details = house.find("div", class_="details-item").text.strip()
    address = house.find("span", class_="comm-address").text.strip()
    price = house.find("span", class_="price-det").text.strip()
    # print("name:{} detai:{} address:{} price:{}".format(name, details, address, price))

    temp["name"] = format_str(name)
    temp["details"] = format_str(details)
    temp["address"] = format_str(address)
    temp["price"]  = format_str(price)
    houses = houses.append(temp, ignore_index=True)

#to_csv 保存csv文件
houses.to_csv(“beijing.csv”, index=False,encoding=‘utf_8_sig’)