[python爬虫]豆瓣电影Top250简单数据分析绘图

一：简介

通过抓取豆瓣电影Top250的数据，分别进行了三个数据统计，分别是：上榜的电影上映的年份，该年份总共上榜的电影数量，数量为0的就没有统计了；各个国家地区出品的电影数量；250部电影的各个类型标签的数量。

二：源代码

#coding=utf-8
import requests
from bs4 import BeautifulSoup
import os, socket, re
import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np

class Spider:
    def __init__(self, url='https://movie.douban.com/top250'):
        self.url = url
        self.header = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36"
        }

    def mkdir(self, path):
        path = path.strip()
        isExists = os.path.exists(os.path.join("D:\mdouban", path))
        if not isExists:
            os.makedirs(os.path.join("D:\mdouban", path))
            os.chdir(os.path.join("D:\mdouban", path))
        else:
            os.chdir(os.path.join("D:\mdouban", path))
        return os.path.abspath('.')

    #获取BeautifulSoup
    def get_soup(self, link):
        html = requests.get(link, headers=self.header)
        html.encoding = html.apparent_encoding
        soup = BeautifulSoup(html.text, 'lxml')
        return soup


if __name__ == '__main__':
    socket.setdefaulttimeout(20)
    spider = Spider()
    path = spider.mkdir('top250')
    print('starting get data from douban...')

    def autolabel(rects, ax, xpos='center'): #设置显示每一个条形图的值
        """
        Attach a text label above each bar in *rects*, displaying its height.

        *xpos* indicates which side to place the text w.r.t. the center of
        the bar. It can be one of the following {'center', 'right', 'left'}.
        """

        xpos = xpos.lower()  # normalize the case of the parameter
        ha = {'center': 'center', 'right': 'left', 'left': 'right'}
        offset = {'center': 0.5, 'right': 0.57, 'left': 0.43}  # x_txt = x + w*off

        for rect in rects:
            height = rect.get_height()
            ax.text(rect.get_x() + rect.get_width() * offset[xpos], 1.01 * height,
                    '{}'.format(height), ha=ha[xpos], va='bottom', size=6.8)

    def drawYearPlot(num_list, name_list):      #绘制X轴为年份，Y轴为电影数量的柱状图
        ind = np.arange(len(name_list))
        fig, ax = plt.subplots()
        ax.set_xlabel('year')
        ax.set_ylabel('numbers')
        ax.set_title('Douban top 250 movie numbers by year')

        rext = ax.bar(ind, num_list, color='b', tick_label=name_list)
        autolabel(rext, ax)
        plt.xticks(np.arange(len(name_list)), rotation=-90, size=7.2)  # 设置X轴坐标的属性

        fig = plt.gcf()
        fig.set_size_inches(15.5, 10.5)  # 设置图片大小
        plt.savefig('D:/mdouban/douban_year.png', dpi=200)  # 保存统计图到本地，必须在show（）方法前调用，否则得到的是一张空白图片,dpi是分辨率
        plt.show()
        plt.close()

    def drawCountryPlot(cry_list):      #绘制X轴为国家地区，Y轴为电影数量的柱状图
        sta = {}
        for i in cry_list:  #统计各个国家的电影数量
            if not sta.__contains__(i):
                sta[i] = 1
            else:
                sta[i] += 1
        num_l = []  #数量
        country_list = [] #国家地区
        for key, values in sta.items():
            country_list.append(key)
            num_l.append(values)

        ind = np.arange(len(country_list))
        fig, ax = plt.subplots()
        ax.set_xlabel('country')
        ax.set_ylabel('numbers')
        ax.set_title('Douban top 250 movie numbers by country')

        rext = ax.bar(ind, num_l, color='b', tick_label=country_list)
        autolabel(rext, ax)
        plt.xticks(np.arange(len(country_list)), size=7.2)  # 设置X轴坐标的属性

        fig = plt.gcf()
        fig.set_size_inches(15.5, 10.5)  # 设置图片大小
        plt.savefig('D:/mdouban/douban_country.png', dpi=200)  # 保存统计图到本地，必须在show（）方法前调用，否则得到的是一张空白图片,dpi是分辨率
        plt.show()
        plt.close()

    def drawTypePlot(typ_list):     #绘制X轴为电影的标签，Y轴为数量的柱状图
        sta = {}
        for i in typ_list:  #统计各个国家的电影数量
            if not sta.__contains__(i):
                sta[i] = 1
            else:
                sta[i] += 1
        num_l = []  #数量
        tp_list = [] #电影类型
        for key, values in sta.items():
            tp_list.append(key)
            num_l.append(values)

        ind = np.arange(len(tp_list))
        fig, ax = plt.subplots()
        ax.set_xlabel('type')
        ax.set_ylabel('numbers')
        ax.set_title('Douban top 250 movie number by type')

        rext = ax.bar(ind, num_l, color='b', tick_label=tp_list)
        autolabel(rext, ax)
        plt.xticks(np.arange(len(tp_list)), size=7.2)  # 设置X轴坐标的属性

        fig = plt.gcf()
        fig.set_size_inches(15.5, 10.5)  # 设置图片大小
        plt.savefig('D:/mdouban/douban_type.png', dpi=200)  # 保存统计图到本地，必须在show（）方法前调用，否则得到的是一张空白图片,dpi是分辨率
        plt.show()
        plt.close()

    #top250共十页
    ys = []     #存储年份
    cs = []     #存储国家地区
    ts = []     #存储电影类别
    #解决matplotlib显示中文乱码问题
    mpl.rcParams['font.sans-serif'] = ['SimHei']  # 指定默认字体 SimHei为黑体
    mpl.rcParams['axes.unicode_minus'] = False  # 用来正常显示负号
    x = 1
    for i in range(1, 11):
        if i == 1:
            url = spider.url
        else:
            url = spider.url + '?start=' + str(25*(i-1)) + '&filter='   #后面9页的链接需要拼接
        main_soup = spider.get_soup(url)
        ol_grid = main_soup.find('ol', class_='grid_view')
        li = ol_grid.find_all('li')

        for l in li:
            em_rank = l.find('em').get_text()

            div_hd = l.find('div', class_='hd')
            a = div_hd.find('a')
            title = a.find('span', class_='title').get_text()

            p_info = l.find('p', class_='').get_text()
            s_c = p_info.split('/')[-2].strip()
            country = s_c.split()[0]        #获取国家地区字段，取第一个
            cs.append(country)

            l_typ = p_info.split('/')[-1].strip().split()   #获取电影类型的数组
            for typ in l_typ:
                ts.append(typ)

            s1 = ''.join(p_info.split()) #去掉字符串中的\xa0
            l_s = s1.split('/')
            if x == 80:
                year = '1961'         #第80的大闹天宫上映了多次，特殊处理
            else:
                year = l_s[-3][-4:]     #电影的上映年份
            x += 1
            ys.append(year)

            div_star = l.find('div', class_='star')
            rating_num = div_star.find('span', class_='rating_num').get_text()
            review = div_star.find_all('span')[3].get_text()

            div_bd = l.find('div', class_='bd')
            q = div_bd.find('span', class_='inq')
            if q != None:   #部分电影是没有短评的，所以需要判断
                quote = q.get_text()
            else:
                quote = '无'
    name_list = []
    sta = {}
    for i in range(1931, 2018):     #柱状图的X轴坐标
        name_list.append(i)
        sta[str(i)] = 0

    for x in ys:        #统计从1931到2017每年在榜单中的电影数量
        sta[x] += 1
    num_list = []
    name_list1 = []
    for key, value in sta.items():
        if value > 0:       #只显示电影数量大于0的
            name_list1.append(str(key))
            num_list.append(value)
    drawYearPlot(num_list, name_list1)
    drawCountryPlot(cs)
    drawTypePlot(ts)

    print('over!')

三：生成的柱状图

douban_country.png

douban_type.png

douban_year.png