爬取思路与小结
- 在查找过程中，查看源代码，bv号可以转化为av号,ss号可以转化为ep号
即可以相互转换，如图，图中一个视频就有ep号，av,bv,cv号，代码中利用了bv号可以转化为av号,ss号可以转化为ep号
- 只能对网页里已有的链接进行爬取，无法爬取大会员视频。
- 打包Python
- pip install pyinstaller
- cd 到bilbili_down.py文件所在位置
- 在cmd终端直接使用 pyinstaller bilbili_down.py
- 这是我已经打包好的：感兴趣的老铁可以试一下功能(第一次打包不小心把我自己的快捷方式打包里面去了，老铁们要打开真正的exe文件啊，不然可能无法保存视频)：https://nmydt.lanzous.com/iMkpUlufosd
代码
import json,requests,os,re,shutil,ssl,time
from concurrent.futures import ThreadPoolExecutor
from lxml import etree
headers = {
'Accept': '*/*',
'Accept-Language': 'en-US,en;q=0.5',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.116 Safari/537.36'
}
params = {
'from': 'search',
'seid': '9698329271136034665'
}
def re_video_info(text, pattern):
'''利用正则表达式匹配出视频信息并转化成json'''
match = re.search(pattern, text)
return json.loads(match.group(1))
def create_folder(aid):
'''创建文件夹'''
if not os.path.exists(aid):
os.mkdir(aid)
def remove_move_file(aid):
'''删除和移动文件'''
file_list = os.listdir('./')
for file in file_list:
if file.endswith('_video.mp4'):
os.remove(file)
pass
elif file.endswith('_audio.mp4'):
os.remove(file)
pass
elif file.endswith('.mp4'):
if os.path.exists(aid + '/' + file):
os.remove(aid + '/' + file)
shutil.move(file, aid)
def BV_move_av(url):
r=requests.get(url)
html = etree.HTML(r.text)
av_url = html.xpath('/html/head/meta[@itemprop="url"]/@content')[0]
aid = re.search('\d+',av_url).group(0)
return aid
def ss_move_ep(url):
r=requests.get(url)
url = "https://www.bilibili.com/bangumi/play/ep"+str(json.loads(re.search('"epList\":(.*?),\"epI',r.text).group(1))[0]['id'])
return url
def download_video_batch(referer_url, video_url, audio_url, video_name, index):
'''批量下载系列视频'''
headers.update({
"Referer": referer_url})
short_name = video_name.split('/')[2]
print("%d.\t视频下载开始：%s" % (index, short_name))
video_content = requests.get(video_url, headers=headers)
print('%d.\t%s\t视频大小：' % (index, short_name),
round(int(video_content.headers.get('content-length', 0)) / 1024 / 1024, 2), '\tMB')
received_video = 0
with open('%s_video.mp4' % video_name, 'ab') as output:
headers['Range'] = 'bytes=' + str(received_video) + '-'
response = requests.get(video_url, headers=headers)
output.write(response.content)
audio_content = requests.get(audio_url, headers=headers)
print('%d.\t%s\t音频大小：' % (index, short_name),
round(int(audio_content.headers.get('content-length', 0)) / 1024 / 1024, 2), '\tMB')
received_audio = 0
with open('%s_audio.mp4' % video_name, 'ab') as output:
headers['Range'] = 'bytes=' + str(received_audio) + '-'
response = requests.get(audio_url, headers=headers)
output.write(response.content)
received_audio += len(response.content)
return video_name, index
def download_video_single(referer_url, video_url, audio_url, video_name):
'''单个视频下载'''
headers.update({
"Referer": referer_url})
print("视频下载开始：%s" % video_name)
video_content = requests.get(video_url, headers=headers)
print('%s\t视频大小：' % video_name, round(int(video_content.headers.get('content-length', 0)) / 1024 / 1024, 2), '\tMB')
received_video = 0
with open('%s_video.mp4' % video_name, 'ab') as output:
headers['Range'] = 'bytes=' + str(received_video) + '-'
response = requests.get(video_url, headers=headers)
output.write(response.content)
audio_content = requests.get(audio_url, headers=headers)
print('%s\t音频大小：' % video_name, round(int(audio_content.headers.get('content-length', 0)) / 1024 / 1024, 2), '\tMB')
received_audio = 0
with open('%s_audio.mp4' % video_name, 'ab') as output:
headers['Range'] = 'bytes=' + str(received_audio) + '-'
response = requests.get(audio_url, headers=headers)
output.write(response.content)
received_audio += len(response.content)
print("视频下载结束：%s" % video_name)
video_audio_merge_single(video_name)
def video_audio_merge_batch(result):
'''使用ffmpeg批量视频音频合并'''
video_name = result.result()[0]
index = result.result()[1]
import subprocess
video_final = video_name.replace('video', 'video_final')
command = 'ffmpeg -i "%s_video.mp4" -i "%s_audio.mp4" -c copy "%s.mp4" -y -loglevel quiet' % (
video_name, video_name, video_final)
subprocess.Popen(command, shell=True)
print("%d.\t视频下载结束：%s" % (index, video_name.split('/')[2]))
def video_audio_merge_single(video_name):
'''使用ffmpeg单个视频音频合并'''
print("视频合成开始：%s" % video_name)
import subprocess
command = 'ffmpeg -i "%s_video.mp4" -i "%s_audio.mp4" -c copy "%s.mp4" -y -loglevel quiet' % (
video_name, video_name, video_name)
subprocess.Popen(command, shell=True)
print("视频合成结束：%s" % video_name)
def batch_download():
'''使用多线程批量下载视频'''
aid = input(
"请输入要下载的视频id（举例：链接https://www.bilibili.com/video/BV1Ke411W71L?p=1中id为1Ke411W71L\nhttps://www.bilibili.com/video/av91748877?p=1中id为91748877，默认为91748877)")
if aid:
if re.search('\D',aid):
aid = BV_move_av('https://www.bilibili.com/video/BV'+aid)
else:
aid = '91748877'
quality = input('请选择清晰度（1代表高清，2代表清晰，3代表流畅），默认高清\t')
if quality == '2':
pass
elif quality == '3':
pass
else:
quality = '1'
acc_quality = int(quality) - 1
ssl._create_default_https_context = ssl._create_unverified_context
url = 'https://www.bilibili.com/video/av{}?p=1'.format(aid)
html = etree.HTML(requests.get(url, params=params, headers=headers).text)
title = html.xpath('//*[@id="viewbox_report"]/h1/span/text()')[0]
print('您即将下载的视频系列是：', title)
create_folder('video')
create_folder('video_final')
pool = ThreadPoolExecutor(3)
res_json = requests.get('https://api.bilibili.com/x/player/pagelist?aid={}'.format(aid)).json()
video_name_list = res_json['data']
print('共下载视频{}个'.format(len(video_name_list)))
for i, video_content in enumerate(video_name_list):
video_name = ('./video/' + video_content['part']).replace(" ", "-")
origin_video_url = 'https://www.bilibili.com/video/av{}'.format(aid) + '?p=%d' % (i + 1)
res = requests.get(origin_video_url, headers=headers)
video_info_temp = re_video_info(res.text, '__playinfo__=(.*?)',r.text).group(1))
catalog = json.loads(re.search('__INITIAL_STATE__=(.*?)\;\(function()',r.text).group(1))
all_num = len(catalog['epList'])
urls=[]
re.search('\D+',url).group(0)
id = int(re.search('\d+',url).group(0))
url_half = re.search('\D+',url).group(0)
[urls.append(url_half+str(id+i)) for i in range(all_num)]
quality = input('请选择清晰度（1代表高清，2代表清晰，3代表流畅），默认高清\t')
if quality == '2':
pass
elif quality == '3':
pass
else:
quality = '1'
acc_quality = int(quality) - 1
create_folder('video')
create_folder('video_final')
pool = ThreadPoolExecutor(3)
for i,ul in enumerate(urls):
r = requests.get(ul)
r.close()
try:
data = json.loads(re.search('__playinfo__=(.*?)',r.text).group(1))
except Exception as e:
break
name = ''.join(catalog['epList'][i]['titleFormat']+' '+catalog['epList'][i]['longTitle'])
video_name = ('./video/' + name).replace(" ", "-")
duration = data['data']['dash']['duration']
quality = data['data']['support_formats'][acc_quality]['display_desc']
video_url = data['data']['dash']['video'][acc_quality]['backupUrl'][0]
audio_url = data['data']['dash']['audio'][acc_quality]['backupUrl'][0]
video_minute = duration // 60
video_second = duration % 60
print('{}.\t当前视频清晰度为{}，时长{}分{}秒'.format(i + 1, quality, video_minute, video_second))
pool.submit(download_video_batch, url, video_url, audio_url, video_name, i + 1).add_done_callback(
video_audio_merge_batch)
pool.shutdown(wait=True)
time.sleep(5)
if os.path.exists(title):
shutil.rmtree(title)
os.rename('video_final', title)
try:
shutil.rmtree('video')
except:
shutil.rmtree('video')
def multiple_download():
'''批量下载多个独立视频'''
aid_str = input(
'请输入要下载的所有视频id，id之间用空格分开\n举例：有5个链接https://www.bilibili.com/video/av89592082、https://www.bilibili.com/video/av68716174、https://www.bilibili.com/video/av87216317、\nhttps://www.bilibili.com/video/av83200644和https://www.bilibili.com/video/av88252843，则输入89592082 68716174 87216317 83200644 88252843\n默认为89592082 68716174 87216317 83200644 88252843\t')
if aid_str:
pass
else:
aid_str = '89592082 68716174 87216317 83200644 88252843'
if os.path.exists(aid_str):
shutil.rmtree(aid_str)
aids = aid_str.split(' ')
quality = input('请选择清晰度（1代表高清，2代表清晰，3代表流畅），默认高清\t')
if quality == '2':
pass
elif quality == '3':
pass
else:
quality = '1'
acc_quality = int(quality) - 1
create_folder(aid_str)
pool = ThreadPoolExecutor(3)
for aid in aids:
pool.submit(single_download, aid, acc_quality)
pool.shutdown(wait=True)
time.sleep(5)
remove_move_file(aid_str)
def single_download(aid, acc_quality):
'''单个视频实现下载'''
origin_video_url = 'https://www.bilibili.com/video/av' + aid
res = requests.get(origin_video_url, headers=headers)
html = etree.HTML(res.text)
title = html.xpath('//*[@id="viewbox_report"]/h1/span/text()')[0]
print('您当前正在下载：', title)
video_info_temp = re_video_info(res.text, '__playinfo__=(.*?)
