python selenium爬取街舞网站视频数据

前言

---

想要做一个街舞网站视频的app，利用爬取的街舞视频存入本地，展现到app上

爬取网站：https://www.vhiphop.com/

Vhiphop网站也是我很喜欢的街舞网站之一，这里聚集了很多我喜欢的舞者视频和教学

爬取代码：

# !/usr/bin/env python
# -*- coding: utf-8 -*-

# 爬取Vhiphop网站

import requests
import importlib
import lxml.html
import os
import sys
importlib.reload(sys)
import json

from selenium import webdriver

video_list = []

def selenium_get_vhiphop_html(http_url):
    # 更改代理ip
    options = webdriver.ChromeOptions()
    options.add_argument("--proxy-server=http://110.73.2.248:8123")

    #获取chromedriver的绝对路径
    driver_path = r'E:\fiddler\chromedriver.exe'
    # 初始化一个driver，并且指定chromedriver的路径
    driver = webdriver.Chrome(executable_path=driver_path)
    # driver = webdriver.Chrome(executable_path=driver_path,, chrome_options=options)
    # 请求网页
    driver.get(http_url)
    # 通过page_source获取网页源代码 热键 ctrl+alt+L
    return driver.page_source


# request请求，但response内容隐藏
def parse_vhiphop_url(http_url):
    #拿到请求头 模拟用户搜索
    headers = {"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36"}
    response = requests.get(http_url,headers=headers,verify=False)
    # 响应首页内容
    return response.text

# 处理html数据，得到所有的数据
def get_vhiphop_video(html_data):
    """分析全网数据，并提取视频信息"""
    # 获得etree对象
    metree = lxml.html.etree
    # 获得解析对象
    parser = metree.HTML(html_data)
    # 解析并获得div列表
    li_list = parser.xpath("//ul[@class='video-lists-box']/li")

    # 遍历div列表
    for li_element in li_list:
        # 拿到一个个li标签
        video_title = li_element.xpath("./a[@class='video-lists-link']/@title")
        # 40个popping信息
        item = {}
        # 获取标题
        item["video_title"] = video_title[0]
        # 获取ul地址，以用查找视频地址
        video_data_html_url = li_element.xpath("./a[@class='video-lists-link']/@href")[0]
        if not video_data_html_url.startswith("http"):
            video_data_html_url = "https://www.vhiphop.com"+video_data_html_url
        # 获取街舞视频播放页面html数据
        # video_data = selenium_get_vhiphop_html(video_data_html_url)
        # 解析html数据
        video_url = video_real_url(video_data_html_url)
        item["video_url"] = video_url
        # video封面图片地址
        video_img_url = li_element.xpath("./a[@class='video-lists-link']/div/img/@data-src")[0]
        item["video_img_url"] = video_img_url


        # 获取街舞类型：popping lacking breaking
        video_type = li_element.xpath("./a[@class='video-lists-link']/div[@class='video-meta-box']/div[@class='video-title-box']/span/text()")[0]
        item["video_type"] = video_type
        video_list.append(item)


def selenium_get_video_html(html_url):
    # 获取chromedriver的绝对路径
    driver_path = r'E:\fiddler\chromedriver.exe'
    # 初始化一个driver，并且指定chromedriver的路径
    driver = webdriver.Chrome(executable_path=driver_path)
    # driver = webdriver.Chrome(executable_path=driver_path,, chrome_options=options)
    # 请求网页
    driver.get(html_url)
    # 通过page_source获取网页源代码 热键 ctrl+alt+L

    return driver.page_source

def video_real_url(video_html_url):
    html_data = selenium_get_video_html(video_html_url)
    # 获得etree对象
    metree = lxml.html.etree
    # 获得解析对象
    parser = metree.HTML(html_data)
    # 解析并获得div
    video_url = parser.xpath("//div[@class='video-container cp']/div/video[@class='vjs-tech']/@src")
    return video_url

def result(http_url):
    item = {}
    html_data = selenium_get_vhiphop_html(http_url)
    get_vhiphop_video(html_data)

def save_video_datas_json_file(video_list):
    """以json格式保存到本地"""
    # 当目录不存在时则创建新目录
    video_dir_name = "./videos"
    if not os.path.exists(video_dir_name):
        # 创建目录
        os.makedirs(video_dir_name)
    # 写入数据 python类型的数据转换为字符串
    video_json_strs = json.dumps(video_list,ensure_ascii=False,indent=2)
    # 写入文件
    video_file = open(video_dir_name+"/video.json","w",encoding="utf-8")
    video_file.write(video_json_strs)
    video_file.close()
    print("总共个数："+len(video_list))
    print("视频数据已经保存成功！谢谢！")

def main():
    # 推荐页
    http_url_index = "https://www.vhiphop.com/"
    # choreograph = 1
    # 编舞
    http_url_choreograph = "https://www.vhiphop.com/videos?sort=1"
    result(http_url_choreograph)
    # popping
    http_url_popping = "https://www.vhiphop.com/videos?sort=2"
    result(http_url_popping)
    # hiphop
    http_url_hiphop = "https://www.vhiphop.com/videos?sort=3"
    result(http_url_hiphop)
    # jazz
    http_url_jazz = "https://www.vhiphop.com/videos?sort=4"
    result(http_url_jazz)
    # locking
    http_url_locking = "https://www.vhiphop.com/videos?sort=5"
    result(http_url_locking)
    # waccking
    http_url_waccking = "https://www.vhiphop.com/videos?sort=6"
    result(http_url_waccking)
    # breaking
    http_url_breaking = "https://www.vhiphop.com/videos?sort=7"
    result(http_url_breaking)
    # house
    http_url_house = "https://www.vhiphop.com/videos?sort=8"
    result(http_url_house)
    # contemporary
    http_url_contemporary = "https://www.vhiphop.com/videos?sort=9"
    result(http_url_contemporary)
    # reggae
    http_url_reggae = "https://www.vhiphop.com/videos?sort=10"
    result(http_url_reggae)
    # freestly
    http_url_freestly = "https://www.vhiphop.com/videos?sort=11"
    result(http_url_freestly)
    # krumping
    http_url_krumping = "https://www.vhiphop.com/videos?sort=12"
    result(http_url_krumping)
    # others
    http_url_others = "https://www.vhiphop.com/videos?sort=13"
    result(http_url_others)
    # 解析网址，请求数据
    # [{视频1},{视频2},{视频3}]
    # 保存文件
    save_video_datas_json_file(video_list)
    # item_popping = get_vhiphop_video(html_data_index)

if __name__ == '__main__':
    main()

爬取的数据以json存入文本

response内容隐藏

request请求相应的response内容不全

检查网页元素可以看到 response内容被隐藏，可以通过selenium直接获取浏览器的Element

缺点

因为需要安装浏览器驱动，所以每次请求网页时都需要打开浏览器，加载时间实在太长。建议采用多线程爬取，等待时间缩短一些

后续优化

等我写好app了在搞