Python Week 2 - 爬取大规模数据

Week 2

第一节练习项目：在 MongoDB 中筛选房源

from bs4 import BeautifulSoup
import requests, pymongo

# Initialization
client = pymongo.MongoClient('localhost', 27017)
xiaozhuDB = client['xiaozhu500']
sheet_tab = xiaozhuDB['sheet_tab']

urls = ['http://bj.xiaozhu.com/search-duanzufang-p{}-0/'.format(str(page)) for page in range(1, 3+1)]  # 3+1 是为了方便理解

'''
for url in urls:
    wb_data = requests.get(url)
    soup = BeautifulSoup(wb_data.text, 'lxml')
    titles = soup.select('span.result_title.hiddenTxt')
    prices = soup.select('span.result_price > i')   # i 才是具体的str(纯数字), 不然会输出为'$xxx/晚'
    for title, price in zip(titles, prices):
        data = {
            'Title': title.get_text(),
            'Price': int(price.get_text())   # 用 int 保存更合理
        }
        sheet_tab.insert_one(data)
'''
＃ 爬完了以上代码注释掉

for item in sheet_tab.find({'Price': {'$gte': 500}}):
    print(item)

第二节练习项目：爬取手机号

from bs4 import BeautifulSoup
import requests, pymongo, time

# initialization
client = pymongo.MongoClient('localhost', 27017)
shoujihaoDB = client['shoujihao']
sheet_tab = shoujihaoDB['sheet_tab']


def get_links_from(pages):
    for i in range(1, pages + 1):
        url_base = 'http://bj.58.com/shoujihao/1/pn{}/'.format(i)  # 个人页(0)无信息, 所以手工定义为全部是商家
        wb_data = requests.get(url_base)
        soup = BeautifulSoup(wb_data.text, 'lxml')
        titles = soup.select('a.t > strong.number')
        links = soup.select('a.t')
        prices = soup.select('b.price')
        for title, link, price in zip(titles, links, prices):
            data = {
                'Title': title.get_text(),
                'Link': link.get('href'),
                'Price': price.get_text()
            }
            sheet_tab.insert_one(data)


get_links_from(116) # 最后一页就是 116

第二周作业：赶集数据爬取

homepagespider.py

# homepagespider.py

from bs4 import BeautifulSoup
import requests


def get_channel_list():
    wb_data = requests.get('http://bj.ganji.com/wu/')
    wb_data.encoding = 'utf-8'
    soup = BeautifulSoup(wb_data.text, 'lxml')
    links = soup.select('dl.fenlei > dt > a')
    for link in links:
        print('http://bj.ganji.com' + link.get('href'))

channel_list = '''
    http://bj.ganji.com/jiaju/
    http://bj.ganji.com/rirongbaihuo/
    http://bj.ganji.com/shouji/
    http://bj.ganji.com/shoujihaoma/
    http://bj.ganji.com/bangong/
    http://bj.ganji.com/nongyongpin/
    http://bj.ganji.com/jiadian/
    http://bj.ganji.com/ershoubijibendiannao/
    http://bj.ganji.com/ruanjiantushu/
    http://bj.ganji.com/yingyouyunfu/
    http://bj.ganji.com/diannao/
    http://bj.ganji.com/xianzhilipin/
    http://bj.ganji.com/fushixiaobaxuemao/
    http://bj.ganji.com/meironghuazhuang/
    http://bj.ganji.com/shuma/
    http://bj.ganji.com/laonianyongpin/
    http://bj.ganji.com/xuniwupin/
    http://bj.ganji.com/qitawupin/
    http://bj.ganji.com/ershoufree/
    http://bj.ganji.com/wupinjiaohuan/
    '''

linkspider.py

# linkspider.py

from bs4 import BeautifulSoup
import requests, pymongo, time, random

# database initialization
client = pymongo.MongoClient('localhost', 27017, connect=False)
ganjiDB = client['ganji1741']
url_list = ganjiDB['url_list']
item_info = ganjiDB['item_info']

header = {
    'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36',
    'Connection':'keep-alive'
}


def get_links_from(channel, page, who_sells='o'):  # 'o' 是个人信息
    url = '{}{}{}'.format(channel, who_sells, page)
    wb_data = requests.get(url, headers=header)
    wb_data.encoding = 'utf-8'  # 不写中文会乱码
    soup = BeautifulSoup(wb_data.text, 'lxml')
    titles = soup.find_all('a', 'ft-tit')  # the selector of titles, link = title.get('href')

    # 排除转转商品
    filters = []
    for title in titles:
        if 'zz-title' in title.get('class'):
            pass
        else:
            filters.append(title)
    titles = filters[:]
    # 过滤完成
    try:
        if soup.find('ul', 'pageLink'):
            for title in titles:
                url_list.insert_one({'url': title.get('href')})
            print('This page is done >', url)
            time.sleep(random.randrange(1, 4))
        elif soup.find('div', 'no-search'):
            print('No further results in this category >', url)
        elif soup.find('p', 'error-tips1'):
            print('You should get some rest... >', url)
            time.sleep(random.randrange(5, 11))
            get_links_from(channel, page)
        else:
            pass
    except TimeoutError as e:
        print('error:', e,  url)
        get_links_from(channel, page)


def get_item_info(url, data=None):
    wb_data = requests.get(url, headers=header)
    wb_data.encoding = 'utf-8'
    soup = BeautifulSoup(wb_data.text, 'lxml')
    no_longer_exist = soup.find('div', 'error')
    if no_longer_exist:
        print('no longer exist...', url)
    else:
        title = soup.find('h1', 'title-name').get_text()
        date = soup.find('i', 'pr-5').text.split('发布')[0].strip() if soup.find('i', 'pr-5') else None
        category = soup.select('ul.det-infor > li > span > a')[0].get_text()
        price = int(soup.select('i.f22.fc-orange.f-type')[0].get_text())
        area = [price.get_text().strip() for price in soup.select('ul.det-infor > li:nth-of-type(3) > a')]
        state = soup.select('ul.second-det-infor.clearfix > li')[0].get_text().split('：')[-1].strip() if soup.find('ul', 'second-det-infor') and soup.select('ul.second-det-infor.clearfix > li')[0].get_text().split('：')[0].strip() == '新旧程度' else None
        url = url
        item_info.insert_one({'title': title, 'date': date, 'category': category, 'price': price, 'area': area, 'state': state, 'url': url})
        print({'title': title, 'date': date, 'category': category, 'price': price, 'area': area, 'state': state, 'url': url})
        time.sleep(random.randrange(1, 5))
        # 规则上面的事情就自己琢磨吧

counter.py

#counter.py

import time
from linkspider import url_list, item_info

while True:
    print(url_list.find().count())
    print(item_info.find().count())
    time.sleep(8)

main.py

# main.py

from multiprocessing import Pool
from linkspider import get_item_info, get_links_from, url_list, item_info
from homepagespider import channel_list


def get_all_links_from(channel):
    for page in range(1, 100):
        get_links_from(channel, page)


# 已经爬取商品信息的 url 会在 url_list 里面, 和之前用 get_all_links_from 获取的 db_urls 做差集, 避免因网络问题而重复爬取
db_urls = [item['url'] for item in url_list.find()]
item_urls = [item['url'] for item in item_info.find()]
x = set(db_urls)
y = set(item_urls)
rest_of_urls = x - y


# 传入 channel_list, 获取商品链接
# if __name__ == '__main__':
#     pool = Pool(len(channel_list.split()))
#     pool.map(get_all_links_from, channel_list.split())

# 传入差集 set, 爬取商品信息
if __name__ == '__main__':
    pool = Pool(processes=50)
    pool.map(get_item_info, rest_of_urls)
    pool.close()

笔记：多用print，别直接就pass，这样会看不出来问题所在。插入数据库时后面要time.sleep(num)，否则会报错，原因不清楚，应该和多进程操作数据库有关。这次的代码主要在于调试和完善，需要耐心咯……