此博客仅为学习交流,如触及第三方利益,请及时联系本人删除
看标题大家可能会有点疑惑,为什么要写这个看起来没什么作用的爬虫,两个音乐软件换着用不香吗?
基于此问题,我以我个人感受罗列了网易和QQ音乐以下几个优缺点:
网易云 | QQ音乐 | |
优点 | 1.推荐功能 2.用户评论 3.有很多优秀的原创音乐人入驻 |
1.非常强大的版权库 2.QQ黏性 |
缺点 | 1.版权问题 2.功能越来越杂 |
1.界面花里胡哨 2.推荐功能有所欠缺 |
我最开始用网易云的原因就是网易云的界面功能简洁,但现在功能太杂,搞得不像是一个音乐软件。
从用户角度上来说,我们使用网易云的目的是为了听音乐(当然也有部分是因为其他原因),良好的评论环境也只能算是锦上添花。
加上前段时间网易发生的暴力裁员事件,不经使我怀疑网易还能否做出令人印象深刻的佳作。
2.1 登录后通过歌单请求找到歌曲ID
2.2 使用Postman测试
2.3 拼接歌曲ID(此地址的数据无法通过爬虫抓取) https://music.163.com/#/song?id=149297
2.4 找到页面渲染时发起到服务器的歌曲信息请求(请求页面数据)https://music.163.com/song?id=149297
2.5 使用Postman测试请求
2.6 根据Postman填入的信息 用爬虫抓取返回的数据
2.7 登录QQ音乐并找到qq音乐的搜索功能接口
2.8 找到qq音乐的歌单接口
2.9 使用用postman测试请求
PS:这里我猜测腾讯判断是否登录 没有用什么黑技术的话 那有可能用的是servlet上下文
2.9.1 点开cookie并将cookie值写入脚本
把这段代码改为自己浏览器里的cookie值
成功返回歌单信息
不知道腾讯是定时刷新cookie还是用了什么可以识别爬虫伪装的技术,爬取歌单列表时,cookie里的部分值可能会变化或者增加几个值,这样返回的响应是:{"retcode":1000,"code":1000,"subcode":0,"msg":"no longin"}
由于我不是非常了解爬虫工程,所以我的解决办法是手动再到网页获取(这时候网页可能会让你再登录一次)
2.9.2 接下来找添加歌曲的接口
通过对比各项参数我们得知需要传入两个重要参数,mid(歌曲ID)和dirid(歌单ID)
考虑到里边一些参数涉及到我个人相关,为了隐私和安全我把参数全都删掉,有兴趣可以根据URL和参数名填写自己的参数
/**
*脚本入口
*
* 代码里的请求参数和cookie修改为自己账号相应的参数
*
* @param songNum 收藏第几首歌
*/
public static void netEasy(Integer songNum){
/**
* 网易云音乐歌单参数
* 拼装请求参数修改为自己想要爬取的指定歌单参数
*/
String url = "https://music.163.com/weapi/v3/playlist/detail?csrf_token=";
HashMap requestParams = new HashMap<>();
requestParams.put("params","");
requestParams.put("encSecKey","");
try {
Document responseDocument = Jsoup.connect(url).data(requestParams).post();
// 解析响应体
String responseData = responseDocument.text();
JSONObject responseJson = JSONObject.parseObject(responseData);
JSONObject playlist = responseJson.getJSONObject("playlist");
/**
* QQ音乐cookie参数
* 添加自己浏览器存储的cookie信息
*/
Map cookies = new HashMap<>();
cookies.put("AMCV_248F210755B762187F000101%40AdobeOrg","");
cookies.put("LW_PsKey","");
cookies.put("LW_TS","");
cookies.put("LW_pid","");
cookies.put("LW_sid","");
cookies.put("LW_uid","");
cookies.put("RK","");
cookies.put("__v3_c_last_10685","");
cookies.put("__v3_c_review_10685","");
cookies.put("__v3_c_visitor","");
cookies.put("_ga","");
cookies.put("_qpsvr_localtk","");
cookies.put("eas_sid","");
cookies.put("mobileUV","");
cookies.put("o_cookie","");
cookies.put("pac_uid","");
cookies.put("pgv_info","");
cookies.put("pgv_pvi","");
cookies.put("pgv_pvid","");
cookies.put("pgv_si","");
cookies.put("player_exist","");
cookies.put("psrf_access_token_expiresAt","");
cookies.put("psrf_musickey_createtime","");
cookies.put("psrf_qqaccess_token","");
cookies.put("psrf_qqopenid","");
cookies.put("psrf_qqrefresh_token","");
cookies.put("psrf_qqunionid","");
cookies.put("ptcz","");
cookies.put("ptisp","");
cookies.put("ptui_loginuin","");
cookies.put("qm_keyst","");
cookies.put("qqmusic_fromtag","");
cookies.put("ts_last","");
cookies.put("ts_refer","");
cookies.put("ts_uid","");
cookies.put("tvfe_boss_uuid","");
cookies.put("ue_skey","");
cookies.put("ue_ts","");
cookies.put("ue_uid","");
cookies.put("ue_uk","");
cookies.put("uin","");
cookies.put("userAction","");
cookies.put("yplayer_open","");
cookies.put("yq_index","");
cookies.put("yqq_stat","");
cookies.put("yq_playdata","");
cookies.put("yq_playschange","");
cookies.put("yqq_stat","");
String songUrl = "https://music.163.com/song?id=";
// 此处是获取QQ音乐的自定义歌单
String songLikeList = "https://c.y.qq.com/splcloud/fcgi-bin/songlist_list.fcg?utf8=1";
String likeList = jsoupByLikeList(songLikeList, cookies);
// 自定义歌单列表
JSONObject jsonObject = JSONObject.parseObject(likeList);
System.out.println(jsonObject.toJSONString());
playlist.getJSONArray("trackIds").forEach(tv -> {
JSONObject trackIdsJson = JSONObject.parseObject(tv.toString());
String id = trackIdsJson.get("id").toString();
/**
* 网易云歌曲参数
* 拼接歌曲地址 https://music.163.com/song?id=
*/
String song = songUrl+id;
Document songDocument = jsoupHtml(song);
Elements songNames = getSongName(songDocument);
songNames.forEach(v -> {
String scriptData = v.toString();
String substring = scriptData.substring(1, scriptData.length());
String reData = substring.substring(substring.indexOf(">") + 1, substring.indexOf("<"));
// 获取网易云歌单中的歌曲名称
String songName = JSONObject.parseObject(reData).get("title").toString();
System.out.println(songName);
// 通过QQ音乐搜索此歌曲
JSONArray jsonArray = qqMusicSearch(songName);
/* jsonArray.forEach(jv -> {
System.out.println(jv);
});*/
// 保证足够多的歌曲搜索结果
if (jsonArray.size() == 10){
JSONObject thisSong = JSONObject.parseObject(jsonArray.get(songNum).toString());
Object mid = thisSong.get("mid");
if (mid != null){
String dirid = "";
// 判断是否成功登录QQ音乐
if ("0".equals(jsonObject.get("retcode").toString())){
JSONArray list = JSONObject.parseArray(jsonObject.get("list").toString());
// 这里我指定我专为网易云创建的歌单
JSONObject songMenu = JSONObject.parseObject(list.get(2).toString());
dirid = songMenu.get("dirid").toString();
// 将歌曲加入歌单
String midId = mid.toString();
String resopnse = addSongByLikeList(midId, dirid, cookies);
System.out.println(resopnse);
}
}
}
});
});
// 根据ID拼接歌曲页面
} catch (Exception e) {
e.printStackTrace();
}
}
public static Document jsoupHtml(String url){
Document document = null;
try {
// 防止反爬虫检测
Thread.sleep(1000);
document = Jsoup.connect(url).userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36")
.get();
} catch (Exception e) {
e.printStackTrace();
}
return document;
}
public static Elements getSongName(Document document){
Element body = document.body();
Elements script = document.select("script[type=application/ld+json]");
return script;
}
public static JSONArray qqMusicSearch(String songName){
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
e.printStackTrace();
}
// w 后面是搜索关键字
String url = "https://c.y.qq.com/soso/fcgi-bin/client_search_cp?w=";
String response = jsoupJson(url);
JSONObject ResponseJson = JSONObject.parseObject(response);
JSONArray jsonArray = JSONObject.parseObject(JSONObject.parseObject(ResponseJson.get("data").toString()).get("song").toString()).getJSONArray("list");
return jsonArray;
}
public static String jsoupJson(String url){
Connection.Response execute = null;
try {
// 防止反爬虫检测
Thread.sleep(1000);
execute = Jsoup.connect(url)
.userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36")
.ignoreContentType(true)
.execute();
} catch (Exception e) {
e.printStackTrace();
}
return execute.body();
}
public static String jsoupByLikeList(String url, Map cookies){
Connection.Response execute = null;
try {
execute = Jsoup.connect(url)
.userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36")
.ignoreContentType(true)
.cookies(cookies)
.execute();
} catch (IOException e) {
e.printStackTrace();
}
return execute.body();
}
// 添加歌曲到指定歌单
public static String addSongByLikeList(String mid,String dirid,Map cookies){
try {
// 防止反爬虫检测
Thread.sleep(1000);
} catch (InterruptedException e) {
e.printStackTrace();
}
// https://c.y.qq.com/splcloud/fcgi-bin/fcg_music_add2songdir.fcg?g_tk=5381
String url = "https://c.y.qq.com/splcloud/fcgi-bin/fcg_music_add2songdir.fcg?g_tk=5381";
Map map = new HashMap<>();
map.put("loginUin","");
map.put("hostUin","");
map.put("format","json");
map.put("inCharset","utf8");
map.put("outCharset","utf-8");
map.put("notice","0");
map.put("platform","");
map.put("needNewCode","");
map.put("uin","");
map.put("midlist",mid);
map.put("typelist","");
map.put("dirid",dirid);
map.put("addtype","");
map.put("formsender","");
map.put("source","");
map.put("r2","");
map.put("r3","");
map.put("utf8","");
map.put("g_tk","");
String text = null;
try {
Document post = Jsoup.connect(url)
.ignoreContentType(true)
.cookies(cookies)
.data(map)
.userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36")
.post();
text = post.text();
} catch (IOException e) {
e.printStackTrace();
}
return text;
}
public static void main(String[] args) {
netEasy(1);
}
{\__/} {\__/}
( ·-·) (·-· )
/ >------------------------------------------------< \
| ☆ |
| ☆ |
| ★ |
| ☆ |
| ☆ |
| |
-------------------------------------