python爬虫学习笔记 3.4 （案例二，动态页面模拟点击）

python爬虫学习笔记 3.4 （案例二，动态页面模拟点击）

案例二：动态页面模拟点击

爬取斗鱼直播平台的所有房间信息：

import time
from selenium import webdriver
import json
import csv

class douyu:
    def __init__(self):
        self.start_url = "https://www.douyu.com/directory/all"
        self.driver = webdriver.Chrome()

    def get_content_list(self):
        li_list = self.driver.find_elements_by_xpath("//ul[@class='layout-Cover-list']/li")
        content_list = []
        for li in li_list:
            item = {}
            item["room_img"] = li.find_element_by_xpath(".//div/a/div/div/img").get_attribute("src")
            item["room_cate"] = li.find_element_by_xpath(".//span[@class='DyListCover-zone']").text
            item["room_title"] = li.find_element_by_xpath(".//h3[@class='DyListCover-intro']").get_attribute("title")
            item["anchor_name"] = li.find_element_by_xpath(".//h2[@class='DyListCover-user']").text
            item["watch_num"] = li.find_element_by_xpath(".//span[@class='DyListCover-hot']").text
            print(item)
            content_list.append(item)
            # 获取下一页的元素
        next_url = self.driver.find_elements_by_xpath("//li[@class=' dy-Pagination-next']")
        next_url = next_url[0] if len(next_url) > 0 else None
        return content_list, next_url

    def save_content_list(self, content_list):
        file_path = "douyu.txt"
        try:
            with open(file_path,'w',encoding="utf-8") as f:
                json.dump(content_list,f,ensure_ascii=False,indent=2)
        except UnicodeEncodeError as e:
            print(e)

    def run(self):
        #1.start_url
        #2.发送请求，获取相应
        self.driver.get(self.start_url)

        time.sleep(10)
        # 3.提取数据，提取下一页的元素
        content_list, next_url = self.get_content_list()
        # 4.保存数据
        self.save_content_list(content_list)
        # 5.点击下一页元素，循环
        while next_url is not None:
            next_url.click()
            time.sleep(10)
            content_list, next_url = self.get_content_list()
            self.save_content_list(content_list)


if __name__ == '__main__':
    douyu = douyu()
    douyu.run()