python——xpath

xpath介绍

节点关系

xpath快捷键chrome

lxml模块使用

把数据存储到字典并保存csv

换一种方法取result的索引——index（）

结果取到了：

豆瓣 top250 翻页爬取 链接、名字、评分、引言

拼接字符串

结果出来有[]，所以可以进行处理索引取值，做非空判断

f’{}'和{}.format()区别

代码总汇：

from lxml import etree
import csv
import requests

# https://movie.douban.com/top250 第一页
# https://movie.douban.com/top250?start=25&filter= 第二页
# https://movie.douban.com/top250?start=50&filter= 第三页
# 所以第一页为：https://movie.douban.com/top250?start=50
# 规律：（page-1）*25

#
doubanUrl = 'https://movie.douban.com/top250?start={}'


def getSource(url):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.128 Safari/537.36'
    }
    resp = requests.get(url, headers=headers)
    resp.encoding = 'utf-8'
    return resp.text


def geteveryItem(source):
    html_element = etree.HTML(source)
    movieItem = html_element.xpath('//div[@class="info"]')
    movieLst = []
    for eachMovie in movieItem:
        movieDict = {}
        # 电影名字
        title = eachMovie.xpath('./div[@class="hd"]/a/span[@class="title"]/text()')
        # 电影别名
        title2 = eachMovie.xpath('./div[@class="hd"]/a/span[@class="other"]/text()')
        # 评分  或者这样写：//div[@class="bd"]/div[@class="star"]/span[@class="rating_num"]
        score = eachMovie.xpath('div[@class="bd"]/div[@class="star"]/span[@class="rating_num"]/text()')
        if score:
            score = score[0]
        else:
            score = ''
        # 导语
        introd = eachMovie.xpath('div[@class="bd"]/p[@class="quote"]/span/text()')
        if introd:
            introd = introd[0]
        else:
            introd = ''
        # 链接url
        links = eachMovie.xpath('./div[@class="hd"]/a/@href')
        if links:
            links = links[0]

        movieDict['名称'] = ''.join(title + title2)
        movieDict['评分'] = score
        movieDict['导语'] = introd
        movieDict['链接'] = links
        movieLst.append(movieDict)
    return movieLst


def saveData(movieLst):
    with open('豆瓣top250', 'w', encoding='utf-8', newline='')as f:
        dwriter = csv.DictWriter(f, fieldnames=['名称', '评分', '导语', '链接'])
        dwriter.writeheader()
        dwriter.writerows(movieLst)


if __name__ == '__main__':
    movieLst = []
    for i in range(10):
        yeshu = doubanUrl.format(i * 25)
        source = getSource(yeshu)  # 在上面取完整的url后在这里爬取html，调用getSource函数
        movieLst += geteveryItem(source)  #把上面取到的完整25页html赋值到解析函数geteveryItem，并将它的值传给movieLst
    saveData(movieLst)