爬取一页商品数据

目标

• 爬取一页商品数据
• 不要推广商品和转转商品
• 在详情页获取信息（类目 | 标题 | 发帖时间 | 价格 | 成色 | 区域 | 浏览量）

环境

• Python 3.5.1
• Windows7 或者 Ubuntu 14.04

结果示例

{
    'areas': ['丰台', '-', '西客站'],
    'datetime': '2016-03-06',
    'price': '500',
    'quality': '95成新',
    'title': '北京实体店出售苹果IPAD迷你AIR三星平板电脑',
    'type': '北京二手平板电脑',
    'views': '6103'
}

代码

# -*- coding: utf-8 -*-

from bs4 import BeautifulSoup
import requests, time
from pprint import pprint

base_url = 'http://bj.58.com/pbdn/0/'

def get_all_links(url):
    """获取一页商品的链接，除了推广和转转"""
    
    urls = []
    data = requests.get(url)
    if data.status_code == 200:
        soup = BeautifulSoup(data.text, 'lxml')
        for tb in soup.select('div#infolist > table.tbimg'):
            for item in tb.find_all("tr", class_=""):
                urls.append(item.find("a", class_="t").get('href'))
        
        return urls

def get_views(url):
    """获取单个商品的浏览量"""
    
    info_id = url.split('?')[0].split('/')[-1].strip('x.shtml')
    interface = 'http://jst1.58.com/counter?infoid={}'.format(info_id)
    headers = {
        'Host': 'jst1.58.com',
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) \
                        Chrome/50.0.2661.102 Safari/537.36',
        'Accept': '*/*',
        'Accept-Language': 'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3',
        'Accept-Encoding': 'gzip, deflate',
        'Referer': url,
        'Cookie': r'id58=c5/ns1djWc+KP2U1DXI4Ag==; city=bj; 58home=bj; ipcity=sh%7C%u4E0A%u6D77%7C0; \
            58tj_uuid=93b726b3-9742-4270-9d11-dcd665bea67b; new_uv=3; myfeet_tooltip=end; \
            als=0; bj58_new_uv=3; bj58_id58s="SUg0UjFZMFI0K3RSMDIyMw=="; \
            sessionid=37cd7e97-40fe-40ac-b903-d6e85ffd8690; __utma=253535702.1612735289.1466129063.1466129063.1466129063.1; \
            __utmc=253535702; __utmz=253535702.1466129063.1.1.utmcsr=baidu|utmccn=(organic)|utmcmd=organic; \
            final_history=26364215035854%2C25245379987755%2C26062681492781; bangbigtip2=1; new_session=0; utm_source=; spm=; \
            init_refer=; bj58_new_session=0; bj58_init_refer="{}"'.format(url),
        'Connection': 'keep-alive',
        'Cache-Control': 'max-age=0' 
    }
    
    resp = requests.get(interface, headers=headers)
    if resp.status_code == 200:
        return resp.text.split('=')[-1]
    else:
        return 0

def get_info(url):
    """获取一页中所有的商品信息"""
    
    url_list = get_all_links(url)
    
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) \
            Chrome/50.0.2661.102 Safari/537.36'
    }
    for link in url_list:
        data = requests.get(link, headers=headers)
        if data.status_code == 200:
            soup = BeautifulSoup(data.text, 'lxml')
            areas = soup.select('.c_25d')
            
            info = {
                'type': soup.select('div.breadCrumb span:nth-of-type(3) a')[0].get_text(),
                'title': soup.select('.mainTitle > h1:nth-of-type(1)')[0].get_text(),
                'datetime': soup.select('.time')[0].get_text(),
                'price': soup.select('.price')[0].get_text(),
                'quality': soup.select('.suUl > li:nth-of-type(2) > div:nth-of-type(2) > span:nth-of-type(1)')[0].get_text(strip=True),
                'areas': list(areas[0].stripped_strings) if len(areas) > 0 else [],
                'views': get_views(data.url)     # 使用重定向后的链接
            }

            pprint(info)
        time.sleep(1)

if __name__ == '__main__':
    get_info(base_url)

总结

• 由于链接中存在重定向的链接，所以需要注意要使用重定向后的链接
• 获取浏览量需要通过其他API接口来进行获取
• 注意网站的反爬虫机制，可以加上请求头或者通过代理IP