python 爬取视频 学习

代码中使用的BeautifulSoup 的select，也可使用 find和find_all

re.findall中req为正则表达式

 

urllib.request.urlretrieve()#将视频复制入文件中

import requests
import  re
import urllib.request
from bs4 import BeautifulSoup
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36'}
req = re.compile(r'srcUrl="(.*?)"')
def download(url):

    soup=BeautifulSoup(requests.get(url,headers=headers).text,'html.parser')
    for i in soup.select('.popularem-ath'):
        a=i.select('a')[0]
        id=a['href']
        title=a.find(class_='popularem-title').text
        new_url='http://www.pearvideo.com/{}'.format(id)
        res = requests.get(new_url, headers=headers, timeout=3).text
        get_video=re.findall(req,res)[0]

        urllib.request.urlretrieve(get_video,'F:\Spider\VideoSpider\{}.mp4'.format(title))

if __name__=='__main__':
    url='http://www.pearvideo.com/popular_4'
    download(url)