python利用requests和lxml库爬取豆瓣top250的电影信息并保存到csv文件中

准备

开发环境：windows 7

开发工具：pycharm

python版本：python 3.7

用到的库：requests，lxml，csv

requests，和lxml库是第三方库需要自行安装，安装方法：pip install requests，pip install lxml

csv库为python自带库不需安装

关键步骤

1. 确定url，设置请求头user-agent，利用requests库请求并得到网页源码
2. 分析返回的源码，利用xpath解析源码得到想要获取的内容并以字典的方式存到列表中
3. 遍历列表将数据写入csv文件中

实现代码

# coding = utf-8
# Author:'Alvin'
# Date: '2018/8/10'
 
import requests
from lxml import html
import csv
 
#step1 确定url，设置请求头user-agent，利用requests库请求并得到网页源码
def get_content():
    url = 'https://movie.douban.com/top250'
    user_agent = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36'
    headers = {
        "User_Agent": user_agent
    }
    response = requests.get(url, headers=headers)
    get_movie_list(response.content)
 
#setp2 分析返回的源码，利用xpath解析源码得到想要获取的内容并以字典的方式存到列表中
def get_movie_list(source):
    selector = html.document_fromstring(source)
    movieList = selector.xpath('//div[@class="info"]')
    movies = []
    for movie in movieList:
        title = movie.xpath('div[@class="hd"]/a/span[@class="title"]/text()')
        other = movie.xpath('div[@class="hd"]/a/span[@class="other"]/text()')
        url = movie.xpath('div[@class="hd"]/a/@href')
        rate_num = movie.xpath('div[@class="bd"]/div[@class="star"]/span[@class="rating_num"]/text()')
        quote = movie.xpath('div[@class="bd"]/p[@class="quote"]/span/text()')
 
        movieDic = {}
        movieDic["title"] = ''.join(title[0] + other[0])
        movieDic["url"] = url[0]
        movieDic["star"] = rate_num[0]
        movieDic["quote"] = quote[0]
        movies.append(movieDic)
 
    write_data(movies)
 
#step3 遍历列表将数据写入csv文件中
def write_data(movies):
    with open("douban.csv", 'w', encoding="UTF-8") as f:
        writer = csv.DictWriter(f, fieldnames=['title', 'star', 'quote', 'url'])
        writer.writeheader()
        for mov in movies:
            print("正在写入：" + mov["title"])
            writer.writerow(mov)
 
 
if __name__ == "__main__":
    get_content()