我的第一个爬虫，爬取北京地区短租房信息

# 导入程序所需要的库。
import requests
from bs4 import BeautifulSoup
import time

# 加入请求头伪装成浏览器
headers = {
    #通过Chrome浏览器复制User-Agent
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36'
}


# 定义判断用户性别的函数
def judgment_sex(class_name):
    if class_name == ['member_ico1']:
        return '女'
    else:
        return '男'


# 获取详细页URL函数
def get_links(url):
    try:
        wb_date = requests.get(url, headers)
    except ConnectionAbortedError:
        print('拒绝连接')
    soup = BeautifulSoup(wb_date.text, 'lxml')
    links = soup.select('#page_list > ul > li > a')
    for link in links:
        herf = link.get("href")
        get_info(herf)


# 获取网页信息函数
def get_info(url):
    wb_date = requests.get(url, headers)
    soup = BeautifulSoup(wb_date.text, 'lxml')
    #通过浏览器copy selector
    tittles = soup.select('div.pho_info > h4')
    addresses = soup.select('span.pr5')
    prises = soup.select('#pricePart > div.day_l > span')
    images = soup.select('#floatRightBox > div.js_box.clearfix > div.member_pic > a > img')
    names = soup.select('#floatRightBox > div.js_box.clearfix > div.w_240 > h6 > a')
    sexs = soup.select('#floatRightBox > div.js_box.clearfix > div.member_pic > div')
    for tittle, address, prise, image, name, sex in zip(tittles, addresses, prises, images, names, sexs):
        date = {
            'tittle': tittle.get_text().strip(),
            'address': address.get_text().strip(),
            'price': prise.get_text(),
            'image': image.get("src"),
            'name': name.get_text(),
            'sex': judgment_sex(sex.get("class"))
        }
        print(date)


if __name__ == '__main__':

    urls = ['http://bj.xiaozhu.com/search-duanzufang-p{}-0/'.format(number) for number in range(1, 14)]
    for single_url in urls:
        get_links(single_url)
        #休眠十秒，防止被封IP
        time.sleep(10)
        
        #缺点：缺少IP管理，采用休眠方法，效率低