java爬虫 酷狗音乐
1.pom.xml 有些依赖与本项目无关
4.0.0
com.zpc
timor
0.0.1-SNAPSHOT
jar
timor
zpc project for Spring Boot
org.springframework.boot
spring-boot-starter-parent
2.0.3.RELEASE
UTF-8
UTF-8
1.8
org.springframework.boot
spring-boot-starter-thymeleaf
org.springframework.boot
spring-boot-starter-web
mysql
mysql-connector-java
runtime
org.springframework.boot
spring-boot-starter-test
test
org.springframework.boot
spring-boot-devtools
true
true
org.apache.activemq
activemq-all
5.15.4
org.springframework.boot
spring-boot-starter-jdbc
mysql
mysql-connector-java
runtime
com.alibaba
druid
1.0.9
org.apache.shiro
shiro-core
1.4.0
org.apache.shiro
shiro-spring
1.4.0
org.springframework.boot
spring-boot-starter-data-jpa
org.projectlombok
lombok
1.16.22
provided
redis.clients
jedis
2.9.0
org.projectlombok
lombok
1.16.22
provided
org.springframework.boot
spring-boot-starter-mail
org.jsoup
jsoup
1.8.3
org.apache.httpcomponents
httpclient
4.5.5
net.sf.json-lib
json-lib
2.4
jdk15
org.springframework.boot
spring-boot-maven-plugin
org.springframework.boot
spring-boot-maven-plugin
true
com.google.cloud.tools
jib-maven-plugin
0.9.0
gcr.io/my-project/image-built-with-jib
package reptile;
import net.sf.json.JSONObject;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.io.IOException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* @Author timor
* @Date 2019/1/9
*/
public class SpiderKugou {
public static String filePath = "G:/music/";
public static String mp3 = "https://wwwapi.kugou.com/yy/index.php?r=play/getdata&callback=jQuery191027067069941080546_1546235744250&"
+ "hash=HASH&album_id=0&_=TIME";
public static String LINK = "https://www.kugou.com/yy/rank/home/PAGE-8888.html?from=rank";
//"https://www.kugou.com/yy/rank/home/PAGE-23784.html?from=rank";
public static void main(String[] args) throws IOException {
for(int i = 1 ; i < 23 ; i++){
String url = LINK.replace("PAGE", i + "");
getTitle(url);
//download("https://www.kugou.com/song/mfy6je5.html");
}
}
public static String getTitle(String url) throws IOException{
HttpGetConnect connect = new HttpGetConnect();
String content = connect.connect(url, "utf-8");
HtmlManage html = new HtmlManage();
Document doc = html.manage(content);
//pc_temp_songlist为歌曲列表的div的class
Element ele = doc.getElementsByClass("pc_temp_songlist").get(0);
//li 是div中的所有歌曲的列表，通过ele.getElementsByTag("li")可以获取所有的li写的详情
Elements eles = ele.getElementsByTag("li");
for(int i = 0 ; i < eles.size() ; i++){
Element item = eles.get(i);
//歌曲名称以及歌手 陈柯宇 - 生僻字
String title = item.attr("title").trim();
//歌曲播放地址 https://www.kugou.com/song/jla02da.html
String link = item.getElementsByTag("a").first().attr("href");
download(link,title);
}
return null;
}
public static String download(String url,String name) throws IOException{
String hash = "";
HttpGetConnect connect = new HttpGetConnect();
String content = connect.connect(url, "utf-8");
HtmlManage html = new HtmlManage();
String regEx = "\"hash\":\"[0-9A-Z]+\"";
// 编译正则表达式
Pattern pattern = Pattern.compile(regEx);
Matcher matcher = pattern.matcher(content);
if (matcher.find()) {
hash = matcher.group();
hash = hash.replace("\"hash\":\"", "");
hash = hash.replace("\"", "");
}
String item = mp3.replace("HASH", hash);
item = item.replace("TIME", System.currentTimeMillis() + "");
System.out.println(item);
String mp = connect.connect(item, "utf-8");
mp = mp.substring(mp.indexOf("(") + 1, mp.length() - 2);
JSONObject json = JSONObject.fromObject(mp);
String playUrl = json.getJSONObject("data").getString("play_url");
System.out.print(playUrl + " == ");
FileDownload down = new FileDownload();
down.download(playUrl, filePath + name + ".mp3");
System.out.println(name + "下载完成");
return playUrl;
}
}
package reptile;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.http.HttpEntity;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.BasicHttpClientConnectionManager;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
/**
* @Author timor
* @Date 2019/1/9
*/
public class HttpGetConnect {
/**
* 获取html内容
* @param url
* @param charsetName UTF-8、GB2312
* @return
* @throws IOException
*/
public static String connect(String url,String charsetName) throws IOException{
BasicHttpClientConnectionManager connManager = new BasicHttpClientConnectionManager();
CloseableHttpClient httpclient = HttpClients.custom()
.setConnectionManager(connManager)
.build();
String content = "";
try{
HttpGet httpget = new HttpGet(url);
RequestConfig requestConfig = RequestConfig.custom()
.setSocketTimeout(5000)
.setConnectTimeout(50000)
.setConnectionRequestTimeout(50000)
.build();
httpget.setConfig(requestConfig);
httpget.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
httpget.setHeader("Accept-Encoding", "gzip,deflate,sdch");
httpget.setHeader("Accept-Language", "zh-CN,zh;q=0.8");
httpget.setHeader("Connection", "keep-alive");
httpget.setHeader("Upgrade-Insecure-Requests", "1");
httpget.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36");
//httpget.setHeader("Hosts", "www.oschina.net");
httpget.setHeader("cache-control", "max-age=0");
CloseableHttpResponse response = httpclient.execute(httpget);
int status = response.getStatusLine().getStatusCode();
if (status >= 200 && status < 300) {
HttpEntity entity = response.getEntity();
InputStream instream = entity.getContent();
BufferedReader br = new BufferedReader(new InputStreamReader(instream,charsetName));
StringBuffer sbf = new StringBuffer();
String line = null;
while ((line = br.readLine()) != null){
sbf.append(line + "");
}
br.close();
content = sbf.toString();
} else {
content = "";
}
}catch(Exception e){
e.printStackTrace();
}finally{
httpclient.close();
}
//log.info("content is " + content);
return content;
}
private static Log log = LogFactory.getLog(HttpGetConnect.class);
}
package reptile;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
/**
* @Author timor
* @Date 2019/1/9
*/
public class HtmlManage {
public Document manage(String html){
Document doc = Jsoup.parse(html);
return doc;
}
public Document manageDirect(String url) throws IOException{
Document doc = Jsoup.connect( url ).get();
return doc;
}
public List manageHtmlTag(Document doc,String tag ){
List list = new ArrayList();
Elements elements = doc.getElementsByTag(tag);
for(int i = 0; i < elements.size() ; i++){
String str = elements.get(i).html();
list.add(str);
}
return list;
}
public List manageHtmlClass(Document doc,String clas ){
List list = new ArrayList();
Elements elements = doc.getElementsByClass(clas);
for(int i = 0; i < elements.size() ; i++){
String str = elements.get(i).html();
list.add(str);
}
return list;
}
public List manageHtmlKey(Document doc,String key,String value ){
List list = new ArrayList();
Elements elements = doc.getElementsByAttributeValue(key, value);
for(int i = 0; i < elements.size() ; i++){
String str = elements.get(i).html();
list.add(str);
}
return list;
}
private static Log log = LogFactory.getLog(HtmlManage.class);
}
package reptile;
import java.io.*;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
/**
* @Author timor
* @Date 2019/1/9
*/
public class FileDownload {
/**
* 文件下载
* @param url 链接地址
* @param path 要保存的路径及文件名
* @return
*/
public static boolean download(String url,String path) throws IOException {
new WriteText().writeToText(url);
new WriteText().writeToText(path);
boolean flag = false;
CloseableHttpClient httpclient = HttpClients.createDefault();
RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(2000)
.setConnectTimeout(2000).build();
HttpGet get = new HttpGet(url);
get.setConfig(requestConfig);
BufferedInputStream in = null;
BufferedOutputStream out = null;
try{
for(int i=0;i<3;i++){
CloseableHttpResponse result = httpclient.execute(get);
System.out.println(result.getStatusLine());
if(result.getStatusLine().getStatusCode() == 200){
in = new BufferedInputStream(result.getEntity().getContent());
File file = new File(path);
out = new BufferedOutputStream(new FileOutputStream(file));
byte[] buffer = new byte[1024];
int len = -1;
while((len = in.read(buffer,0,1024)) > -1){
out.write(buffer,0,len);
}
flag = true;
break;
}else if(result.getStatusLine().getStatusCode() == 500){
continue ;
}
}
}catch(Exception e){
e.printStackTrace();
flag = false;
}finally{
get.releaseConnection();
try{
if(in != null){
in.close();
}
if(out != null){
out.close();
}
}catch(Exception e){
e.printStackTrace();
flag = false;
}
}
return flag;
}
private static Log log = LogFactory.getLog(FileDownload.class);
}
package reptile;
import java.io.*;
/**
* @Author timor
* @Date 2019/1/9
* 写入歌曲信息到本地text G:\music\music_info
*/
public class WriteText {
public void writeToText(String musicInfo) throws IOException {
String path = "G:\\music\\music_info\\music_info.txt";
File file = new File(path);
if(!file.exists()){
file.getParentFile().mkdirs();
}
file.createNewFile();
// write
FileWriter fw = new FileWriter(file, true);
BufferedWriter bw = new BufferedWriter(fw);
//写入到txt并自动换行
bw.write(musicInfo+"\r\n");
bw.flush();
bw.close();
fw.close();
// read
FileReader fr = new FileReader(file);
BufferedReader br = new BufferedReader(fr);
String str = br.readLine();
}
}
package com.zpc.timor;
import reptile.WriteText;
import java.io.IOException;
/**
* 写入txt测试类
*/
public class test {
public static void main(String[] args) throws IOException {
new WriteText().writeToText("12376\r\n");
}
}