Python网络爬虫与信息提取！爬虫批量搜索音乐并下载！
目标
将想要下载的歌曲名字存在列表中，批量搜索并下载代码。
准备
因为webdriver打开网页缓慢的原因，我考虑使用selenium控制已经打开的chrome网页，毕竟直接用chrome搜索歌曲和打开网页还是很快的。首先在pycharm中打开终端，输入以下命令切换到谷歌浏览器目录下：
cd \d C:\Program Files (x86)\Google\Chrome\Application 1
执行命令打开谷歌浏览器并保存配置在本地，这里路径可以选择和代码同级目录：
chrome.exe --remote-debugging-port=9222 --user-data-dir="e:\py_code\Reptile" 1
执行后会打开谷歌浏览器 ，在地址栏输入我们的网址：
http://www.gequdaquan.net/gqss/index.html
OK，准备工作完成！接下来交给脚本干活。
代码设计
网页驱动的配置：
chrome_options = Options()
# chrome_options.add_experimental_option("debuggerAddress", "127.0.0.1:9222")
chrome_options.debugger_address = "127.0.0.1:9222"
chrome_driver = "chromedriver.exe"
driver = webdriver.Chrome(chrome_driver, chrome_options=chrome_options)
这没啥好讲的，固定格式和端口号，具体参照谷歌中自动控制说明。
网页静音
因为前一篇文章说了下载歌曲需要点击播放，所以静音好点：
try:
driver.find_element_by_xpath("//a[@class='player-btn btn-quiet']").click()
except:
pass
这里用try except框架，因为如果用户点击了静音，再一次点击是静音按钮的class属性会发生变化从而找不到对应的xpath会引发异常，所以如果异常直接跳过即可。
初始化
这里初始化搜索的所有歌曲名字，本地保存路径和歌曲下载地址的列表：
search_name = ["万有引力","苦笑"]
savaer_path = "D://music//"
list = []
批量获取url
下面这段代码调用我们自己写的getMusicUrl函数获取每个搜索名字对应歌曲的URL：
print("开始获取url")
for name in search_name:
music_url = getMusicUrl(driver,name)
list.append(music_url)
函数具体实现和上一篇博客基本相同，不做累述：
def getMusicUrl(driver, search_name):
print("打开搜素框")
driver.find_element_by_xpath("//span[@data-action = \"search\"]").click() # 点击按钮
print("搜索音乐")
getXpath(driver, "//div[@class='search-group']/input[@id='search-wd']").clear()
getXpath(driver, "//div[@class='search-group']/input[@id='search-wd']").send_keys(search_name)
getXpath(driver, "//div[@class='search-group']/button[@class='search-submit']").submit()
time.sleep(5)
print("播放音乐")
# 因为搜索会重新加载界面，如果获取不到按钮控件则不能调用点击函数会抛出异常
flag = True
while flag is True:
try:
flag = False
target = getXpath(driver,"//div[@class='list-item'][1]")
ActionChains(driver).move_to_element(target).perform()
getXpath(driver, "//div[@class='list-item'][1]/span[@class='music-name']/div[@class='list-menu']/span[@class='list-icon icon-play']").click()
except:
flag =True
print("获取地址")
#music_url = str(driver.find_element_by_xpath("//audio").get_attribute("src"))
music_url = str(getXpath(driver, "//audio").get_attribute("src"))
print(music_url)
print("关闭音乐")
getXpath(driver, "//a[@class='player-btn btn-play btn-state-paused']").click()
return music_url
批量下载
下面这段代码遍历url列表，并批量下载：
print("开始下载")
num = 0
for _url in list:
DownloadFile(_url, savaer_path, search_name[num] + ".mp3")
num+=1
下载代码，上一篇博客已经详细介绍不做累述：
def DownloadFile(mp3_url, save_url,file_name):
try:
if mp3_url is None or save_url is None or file_name is None:
print('参数错误')
return None
# 文件夹不存在，则创建文件夹
folder = os.path.exists(save_url)
if not folder:
os.makedirs(save_url)
# 读取MP3资源
res = requests.get(mp3_url,stream=True)
# 获取文件地址
file_path = os.path.join(save_url, file_name)
print('开始写入文件：', file_path)
# 打开本地文件夹路径file_path，以二进制流方式写入，保存到本地
with open(file_path, 'wb') as fd:
for chunk in res.iter_content():
fd.write(chunk)
print(file_name+' 成功下载！')
except:
print("程序错误")
运行验证
完整代码
# coding=utf-8
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.action_chains import ActionChains
import requests
import os
import time
def DownloadFile(mp3_url, save_url,file_name):
try:
if mp3_url is None or save_url is None or file_name is None:
print('参数错误')
return None
# 文件夹不存在，则创建文件夹
folder = os.path.exists(save_url)
if not folder:
os.makedirs(save_url)
# 读取MP3资源
res = requests.get(mp3_url,stream=True)
# 获取文件地址
file_path = os.path.join(save_url, file_name)
print('开始写入文件：', file_path)
# 打开本地文件夹路径file_path，以二进制流方式写入，保存到本地
with open(file_path, 'wb') as fd:
for chunk in res.iter_content():
fd.write(chunk)
print(file_name+' 成功下载！')
except:
print("程序错误")
def getMusicUrl(driver, search_name):
# opt = webdriver.ChromeOptions() # 创建浏览器
# #opt.add_argument('--headless') #无窗口模式
# opt.add_argument("--mute-audio") # 静音
# driver = webdriver.Chrome(options=opt) # 创建浏览器对象
#
# print("打开网页")
# driver.get('http://www.gequdaquan.net/gqss/index.html') # 打开网页
print("打开搜素框")
driver.find_element_by_xpath("//span[@data-action = \"search\"]").click() # 点击按钮
print("搜索音乐")
getXpath(driver, "//div[@class='search-group']/input[@id='search-wd']").clear()
getXpath(driver, "//div[@class='search-group']/input[@id='search-wd']").send_keys(search_name)
getXpath(driver, "//div[@class='search-group']/button[@class='search-submit']").submit()
time.sleep(5)
print("播放音乐")
# 因为搜索会重新加载界面，如果获取不到按钮控件则不能调用点击函数会抛出异常
flag = True
while flag is True:
try:
flag = False
target = getXpath(driver,"//div[@class='list-item'][1]")
ActionChains(driver).move_to_element(target).perform()
getXpath(driver, "//div[@class='list-item'][1]/span[@class='music-name']/div[@class='list-menu']/span[@class='list-icon icon-play']").click()
except:
flag =True
print("获取地址")
#music_url = str(driver.find_element_by_xpath("//audio").get_attribute("src"))
music_url = str(getXpath(driver, "//audio").get_attribute("src"))
print(music_url)
print("关闭音乐")
getXpath(driver, "//a[@class='player-btn btn-play btn-state-paused']").click()
return music_url
def getXpath(driver,path):
flag = True
while flag is True:
try:
flag = False
driver.find_element_by_xpath(path)
except:
flag = True
return driver.find_element_by_xpath(path)
if __name__ == '__main__':
search_name = ["万有引力","苦笑"]
savaer_path = "D://music//"
list = []
chrome_options = Options()
# chrome_options.add_experimental_option("debuggerAddress", "127.0.0.1:9222")
chrome_options.debugger_address = "127.0.0.1:9222"
chrome_driver = "chromedriver.exe"
driver = webdriver.Chrome(chrome_driver, chrome_options=chrome_options)
try:
driver.find_element_by_xpath("//a[@class='player-btn btn-quiet']").click()
except:
pass
print("开始获取url")
for name in search_name:
music_url = getMusicUrl(driver,name)
list.append(music_url)
driver.quit()
print("开始下载")
num = 0
for _url in list:
DownloadFile(_url, savaer_path, search_name[num] + ".mp3")
num+=1
源码文件加群：1136192749