Python Week 1 - 爬取网页信息

Week 1

第一节练习项目：动手做自己的网页

    
    
    


    

        
        

            
Home
            
Site
            
Other
        
    
    

        
The blah
        

        

            

            

            

        
        

            Wa! Wa! Wa! Wa! Wa!
        
        
Wow!
    
    

        
©MUGGLECODING
    

笔记：CSS 写得好就是省事，HTML 和 CSS 互相促进结构化……之前对其有大致了解，在此不深究

第二节练习项目：爬取商品信息

from bs4 import BeautifulSoup

with open('/Users/arischow/Downloads/Plan-for-combating-master/week1/1_2/1_2answer_of_homework/1_2_homework_required/index.html', 'r', encoding='utf-8') as wb_data:
    Soup = BeautifulSoup(wb_data, 'lxml')
    titles = Soup.select('div.caption > h4 > a')
    prices = Soup.select('div.caption > h4.pull-right')
    descs = Soup.select('div.caption > p')
    reviews = Soup.select('div.ratings > p.pull-right')
    rates = Soup.select('div.ratings > p:nth-of-type(2)')
    for title, price, desc, review, rate in zip(titles, prices, descs, reviews, rates):
        data = {
            'title': title.get_text(),
            'price': price.get_text(),
            'desc': desc.get_text(),
            'review': review.get_text(),
            'rate': len(rate.find_all('span', 'glyphicon glyphicon-star'))
        }
        print(data)

笔记：看手册，find_all 返回的会是一个 list，len 的值就代表有多少颗星星……

第三节练习项目：爬取租房信息

from bs4 import BeautifulSoup
import requests
import re

searchPage = ['http://bj.xiaozhu.com/search-duanzufang-p1-0/']
searchPages = ['http://bj.xiaozhu.com/search-duanzufang-p{}-0/'.format(str(i)) for i in range(1,14)]
homePages = []


def get300():
    for url in searchPages:
        wb_data = requests.get(url)
        soup = BeautifulSoup(wb_data.text, 'lxml')
        links = soup.select('ul > li')
        for link in links:
            link = link.find_all(href=re.compile('fangzi'))
            for l in link:
                homePages.append(l.get('href'))
    return homePages

def get300info():
    for url in homePages:
        wb_data = requests.get(url)
        soup = BeautifulSoup(wb_data.text, 'lxml')
        titles = soup.select('body > div.wrap.clearfix.con_bg > div.con_l > div.pho_info > h4 > em')
        addrs = soup.select('body > div.wrap.clearfix.con_bg > div.con_l > div.pho_info > p > span.pr5')
        firstImgs = soup.select('#curBigImage')
        prices = soup.select('#pricePart > div.day_l')
        ownerImgs = soup.select('#floatRightBox > div.js_box.clearfix > div.member_pic > a > img')
        ownerNames = soup.select('#floatRightBox > div.js_box.clearfix > div.w_240 > h6 > a')
        genders = soup.select('#floatRightBox > div.js_box.clearfix > div.member_pic')

        for title, addr, firstImg, price, ownerImg, ownerName, gender in zip(titles, addrs, firstImgs, prices, ownerImgs, ownerNames, genders):
            data = {
                'title': title.get_text(),
                'addr': addr.get_text().strip(),
                'firstImg': firstImg.get('src'),
                'price': price.get_text(),
                'ownerImg': ownerImg.get('src'),
                'ownerName': ownerName.get_text(),
            }
            if gender.find_all(class_='member_ico1') != []:
                data['gender'] = 'female'
            else:
                data['gender'] = 'male'
            print(data)


get300()
get300info()

笔记：这个也是很久前就写好了，现在回想起来，感觉直接跑就可以了，根本不需要用 def 封装，房东性别不同，div.member_pic 所对应的 class 也不同，所以写个简单的 if 判断就可以搞定。

第一周实战作业：爬取一页商品数据

from bs4 import BeautifulSoup
import requests, time

url = 'http://bj.58.com/pbdn/0/'
header = {
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.94 Safari/537.36'
}
wb_data = requests.get(url, headers=header)
soup = BeautifulSoup(wb_data.text, 'lxml')
titles = soup.select('tr > td.t > a.t')  # 未过滤推广/转转商品
filters = []

for title in titles:
    if title.get('data-addtype') == None and title.get('onclick') == None:
        filters.append(title)
titles = filters[:]  # 过滤完毕, 复制列表

itemlinks = []
counter_base = 'http://jst1.58.com/counter?infoid='

for title in titles:
    itemlinks.append(title.get('href')[:title.get('href').find('?psid')])

for link in itemlinks:
    item_url = link
    item_url_digits_only = item_url[item_url.rfind('/') + 1 : -7]  # counter infoid 需要
    counter_url = counter_base + item_url_digits_only
    counter_header = {'Referer': item_url}
    item_data = requests.get(item_url, headers=header)
    item_soup = BeautifulSoup(item_data.text, 'lxml')

    # 爬取内容
    item_categories = item_soup.select('#header > div.breadCrumb.f12 > span:nth-of-type(3) > a')
    item_titles = item_soup.select('div.per_ad_left > div.col_sub.mainTitle > h1')
    item_published_dates = item_soup.select('ul.mtit_con_left.fl > li.time')
    item_prices = item_soup.select('div.col_sub.sumary > ul > li > div.su_con > span.price.c_f50')
    item_purities = item_soup.select('div.col_sub.sumary > ul > li:nth-of-type(2) > div.su_con > span')
    item_zones = item_soup.select('div.col_sub.sumary > ul > li:nth-of-type(3) > div.su_con > span')
    item_views = str(requests.get(counter_url, headers=counter_header).content)  #  将 byte 转化为 str 作字符串处理
    item_views = item_views[item_views.find('total=') + 6 : -1]  # 提取浏览量

    for c, t, pd, pr, pu, z in zip(item_categories, item_titles, item_published_dates, item_prices, item_purities, item_zones):
        zones = list(z.stripped_strings)  # print 一下就知道, 连那个 '-' 都在 list 里面, 没用的东西自然要去掉
        zones = [x for x in zones if x != '-']  # 不是 '-' 的加进 list, 换而言之就是 带 '-' 的去掉了
        data = {
            'Category': c.get_text(),
            'Title': t.get_text(),
            'Published Date': pd.get_text(),
            'Price': pr.get_text(),
            'Purity': pu.get_text().strip(),
            'Zone': list(z.stripped_strings),
            'Views': item_views,
            #'URL': item_url
        }
        print(data)

笔记：推广商品在 里面会带 'data-addtype=level2' 这个属性，而转转商品则会有 'onclick=xxxxxxxxx' 属性，而正常商品是不会有这两个属性的，所以写个判断把它们排除。item_views 返回的这堆字符串是要经过自己处理的，合理利用 find / lfind / rfind 处理起来会非常方便，另外header带着Referer 去爬 counter 是必须的，否则 js 返回的值始终为 0。多个地区的问题参考 1-2 时的代码，用stripped_strings，这里解决是解决了，不过感觉代码还可以精简。