Python爬虫-爬取豆瓣电影Top250

小菜鸟一枚，这是第五个爬虫吧，记录一下

#!usr/bin/env python3
# -*- coding:utf-8-*-

import requests
from bs4 import BeautifulSoup
import re
import csv

header = {
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
        'Accept-Encoding': 'gzip, deflate, br',
        'Accept-Language': 'zh-CN,zh;q=0.8',
        'Connection': 'keep-alive',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 '
                      '(KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36'
}

url_list = ['https://movie.douban.com/top250?start=%d' % index for index in range(0, 250, 25)]

# url = 'https://movie.douban.com/top250?start=0'


def movie_list(url):
    response = requests.get(url, header)
    response.encoding = 'utf-8'
    html = BeautifulSoup(response.text, 'html.parser')
    data = html.find('ol', {'class': 'grid_view'})
    m_list = data.find_all('li')
    movies = []
    for m in m_list:
        rank = m.find('em').get_text()  # 排名
        m_name = m.find('img')['alt']  # 获取电影名字
        info = m.find('p').get_text()
        director = re.findall('导演:\s(.*?)\s', info)[0]  # 导演
        starring = re.findall('主演:\s(.*?)\s', info) # 主演
        if len(starring) == 0:
            starring = '佚名'  # 因为豆瓣显示不全，所以。。
        else:
            starring = starring[0]
        year = re.search(r'\d{4}', info).group()  # 获取年份
        area_list = re.findall('\s/\s(.*?)\s/\s', info)
        # area = re.search(r'\/\n{*}\n\/', info)
        if len(area_list) > 1:
            area = area_list[1]
        else:
            area = area_list[0]
        grade = m.select('span.rating_num')[0].get_text() # 评分
        quote_l = m.select('span.inq') # 简介？
        if len(quote_l) == 0:
            quote = ''
        else:
            quote = quote_l[0].get_text()

        tup = (rank, m_name, director, starring, year, area, grade, quote)
        movies.append(tup)
    return movies

# 将内容保存到csv文件肿
def save_data():
    headers = ['排名', '名字', '导演', '主演', '年份', '地区', '评分', '简介']
    with open('/Users/mocokoo/Documents/py_file/douban_movie_top250.csv', encoding='UTF-8', mode='w') as f:
        f_csv = csv.writer(f)
        f_csv.writerow(headers)
        for url in url_list:
            data_list = movie_list(url)
            for data in data_list:
                f_csv.writerow(data)


if __name__ == '__main__':
    save_data()