爬取58同城

写了一上午，爬了一下午，心累。

1. 采用了多进程爬虫mutliprocessing的Pool
2. 采用mongdb数据库存储数据
3. 采用requests + BeautifulSoup的路线
   思路：
4. 获得各个分类的链接，获得每个物品的链接，存储
5. 从数据库中取链接，获得每个物品的具体信息，存储
6. 另外写一个python程序或者从mongdb shell访问数据库，得到进度
7. 断点续传的功能，在两个数据库中设置一个关联值（这里采用url），然后分别把数据导入一个set，两集合相减就得到接下来要下载的信息

import pymongo,time,requests
from bs4 import BeautifulSoup
from multiprocessing import Pool

client = pymongo.MongoClient('localhost',27017)
tongcheng = client['tongcheng']
url_list = tongcheng['url_list']    #创建了两个collections
info_list = tongcheng['info_list']

headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:53.0) Gecko/20100101 Firefox/53.0'
}    #模拟浏览器访问网站


start_url = 'http://bj.58.com/sale.shtml'
channel_links = []
def get_channel_url(url):
    r = requests.get(url,headers=headers)
    r.raise_for_status()
    soup = BeautifulSoup(r.text,'lxml')
    channels = soup.select('ul.ym-mainmnu > li.ym-tab > ul.ym-submnu > li > b > a')
    for channel in channels:
        channel_links.append('http://bj.58.com'+channel.get('href'))


def get_item_url(channel,page,who_sell=0):
    url = '{}/{}/pn{}/'.format(channel,str(who_sell),str(page))
    wb_data = requests.get(url,timeout=30,headers=headers)
    time.sleep(2)
    wb_data.raise_for_status()
    soup = BeautifulSoup(wb_data.text,'lxml')
    if soup.find('td'):
        items = soup.select('a.t')
        for item in items:
            link = item.get('href').split('?')[0]
            if 'zhuanzhuan' in link:
                url_list.insert_one({'url':link})
                print(link)



def get_pages_url(channel):
    for i in range(1,100):
        get_item_url(channel,i)

def get_item_info(url):
    wb_data = requests.get(url,timeout = 30,headers = headers)
    time.sleep(1)
    wb_data.raise_for_status()
    soup = BeautifulSoup(wb_data.text,'lxml')
    title = soup.find('head').find('title').text
    price = soup.select('span.price_now > i')
    area = soup.select('div.palce_li > span > i')
    view = soup.select('span.look_time')
    info = {
        'title':title.strip(),
        'price':price[0].text if price else None,
        'area':area[0].text.strip() if area else None,
        'view':view[0].text.strip()[:-3] if view else None
    }
    info_list.insert_one(info)
    print(info)


if __name__ == '__main__':
    pool = Pool()    #创建进程池
    get_channel_url(start_url)
    pool.map(get_pages_url,channel_links)     #采用多进程爬取
    pool.map(get_item_info,link['url'] for link in url_list.find())

import pymongo,time

client = pymongo.MongoClient('localhost',27017)
tongcheng = client['tongcheng']
url_list = tongcheng['url_list']
info_list = tongcheng['info_list']

while url_list.find().count() > info_list.find().count():
    print(url_list.find().count()+info_list.find().count())
    time.sleep(5)     #定时得到数据库中的个数

# ================================================= < <链接去重 > > =====================================================

# 设计思路：
# 1.分两个数据库，第一个用于只用于存放抓取下来的 url (ulr_list)；第二个则储存 url 对应的物品详情信息(item_info)
# 2.在抓取过程中在第二个数据库中写入数据的同时，新增一个字段(key) 'index_url' 即该详情对应的链接
# 3.若抓取中断，在第二个存放详情页信息的数据库中的 url 字段应该是第一个数据库中 url 集合的子集
# 4.两个集合的 url 相减得出圣贤应该抓取的 url 还有哪些


db_urls = [item['url'] for item in url_list.find()]     # 用列表解析式装入所有要爬取的链接
index_urls = [item['url'] for item in item_info.find()] # 所引出详情信息数据库中所有的现存的 url 字段
x = set(db_urls)                                        # 转换成集合的数据结构
y = set(index_urls)
rest_of_urls = x-y                               
       # 相减