Python 学习——每天写点小东西-4

今天的爬虫比较简单，就不做详细介绍。爬取的是某网站的租房信息。

from bs4 import BeautifulSoup
import requests
import time


base_urls = ['http://bj.xiaozhu.com/search-duanzufang-p{}-0/'.format(str(i)) for i in range(14)]
headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_3) \
    AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36'}


def get_urls(url):
    r = requests.get(url, headers=headers)
    soup = BeautifulSoup(r.text, 'lxml')
    urls = soup.select('a[class="resule_img_a"]')
    for url in urls:
        href = url.get('href')
        get_details(href)
        time.sleep(1)


def get_details(url):
    r = requests.get(url, headers=headers)
    soup = BeautifulSoup(r.text, 'lxml')

    title = soup.select('div.pho_info > h4 > em')
    addr = soup.select('p > span.pr5')
    price = soup.select('div.day_l > span')
    image = soup.select('div.pho_show_l > div > div > img')
    owner_img = soup.select('div.member_pic > a > img')
    sex = soup.select('div.member_pic > div.member_ico1')
    name = soup.select('div.w_240 > h6 > a')

    for t, a, p, i, o, n in zip(title, addr, price, image, owner_img, name):
        data = {
            'title': t.get_text(),
            'address': ''.join(a.get_text().split()),
            'price': p.get_text(),
            'img': i.get('src'),
            'owner_img': o.get('src'),
            'name': n.get_text()
        }

    if len(sex):
        data['sex'] = 'female'
    else:
        data['sex'] = 'male'

    # return data
    print(data)

for base_url in base_urls:
    get_urls(base_url)
    time.sleep(2)

自己电脑网速太慢一直爬不完，最后使用AWS主机才爬完300多条记录。