15--jQuery插件大全-- 使用jsoup爬取酷我音乐和微博热搜数据
目录
歌曲来自于酷我音乐热歌榜
热搜数据来自于微博热搜
库我音乐前台代码如下：
库我音乐后台代码如下：
Servlet代码：
model代码：
HTTPUtils工具类
微博热搜前台代码如下：
微博热搜后台代码如下：
Servlet代码：
HotModel代码：
URLHandle代码：
HotParse代码：
HTTPUtils代码：
歌曲来自于酷我音乐热歌榜
热搜数据来自于微博热搜
库我音乐前台代码如下：
bootstrapTable
欢迎来到库我音乐
库我音乐后台代码如下：
Servlet代码：
package cn.campusshop.controller;
import java.io.IOException;
import javax.servlet.ServletException;
import javax.servlet.annotation.WebServlet;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.util.EntityUtils;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import cn.campusshop.model.MusicModel;
import cn.campusshop.utils.HTTPUtils;
/**
* Servlet implementation class MusicController
*/
@WebServlet("/music")
public class MusicController extends HttpServlet {
private static final long serialVersionUID = 1L;
/**
* @see HttpServlet#HttpServlet()
*/
public MusicController() {
super();
// TODO Auto-generated constructor stub
}
/**
* @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse response)
*/
protected void doGet(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
HttpClient client = HttpClientBuilder.create().build();
System.out.println("客户端生成完毕.");
String rid = request.getParameter("rid");
System.out.println("rid:" + rid);
String url = "http://kuwo.cn/url?format=mp3&rid=" + rid + "&response=url&type=convert_url3";
// 获取响应资源
HttpResponse responseM = HTTPUtils.getHtml(client, url);
// 获取响应状态码
int statusCode = responseM.getStatusLine().getStatusCode();
System.out.println(statusCode);
MusicModel model = new MusicModel();
if (statusCode == 200) {// 200表示成功
String entity = EntityUtils.toString(responseM.getEntity(), "utf-8");
// model = (MusicModel) JSON.parse(entity);
JSONObject modelJ = JSON.parseObject(entity);
String url1 = modelJ.getString("url");
String msg = modelJ.getString("msg");
String code = modelJ.getString("code");
model.setCode(code);
model.setMsg(msg);
model.setUrl(url1);
} else {
return;
}
String jsonString = JSON.toJSONString(model);
response.setContentType("application/json;charset=utf-8");
response.setHeader("Access-Control-Allow-Origin", "*");
response.getWriter().write(jsonString);
System.out.println("整个结果输出完毕，程序结束.");
}
/**
* @see HttpServlet#doPost(HttpServletRequest request, HttpServletResponse response)
*/
protected void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
// TODO Auto-generated method stub
doGet(request, response);
}
}
model代码：
/**
* Copyright (C) 2019 Baidu, Inc. All Rights Reserved.
*/
package cn.campusshop.model;
/**
* @author zhulang
* @version 创建时间：2019年9月5日 下午11:26:41
*/
public class MusicModel {
private String code;// 状态
private String msg;// 信息音乐名
private String url;// 链接
/**
* @return the code
*/
public String getCode() {
return code;
}
/**
* @param code the code to set
*/
public void setCode(String code) {
this.code = code;
}
/**
* @return the msg
*/
public String getMsg() {
return msg;
}
/**
* @param msg the msg to set
*/
public void setMsg(String msg) {
this.msg = msg;
}
/**
* @return the url
*/
public String getUrl() {
return url;
}
/**
* @param url the url to set
*/
public void setUrl(String url) {
this.url = url;
}
/*
* (non-Javadoc)
*
* @see java.lang.Object#toString()
*/
@Override
public String toString() {
return "HotModel [code=" + code + ", msg=" + msg + ", url=" + url + "]";
}
}
HTTPUtils工具类
/**
* Copyright (C) 2019 Baidu, Inc. All Rights Reserved.
*/
package cn.campusshop.utils;
import java.io.IOException;
/**
* @author zhulang
*@version 创建时间：2019年9月5日 下午11:44:39
*/
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.HttpVersion;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.message.BasicHttpResponse;
public class HTTPUtils {
public static HttpResponse getHtml(HttpClient client, String url) {
// 获取响应文件，即HTML，采用get方法获取响应数据
HttpGet getMethod = new HttpGet(url);
HttpResponse response = new BasicHttpResponse(HttpVersion.HTTP_1_1, HttpStatus.SC_OK, "OK");
try {
// 通过client执行get方法
response = client.execute(getMethod);
} catch (IOException e) {
e.printStackTrace();
} finally {
// getMethod.abort();
}
return response;
}
}
微博热搜前台代码如下：
bootstrapTable
微博热搜后台代码如下：
Servlet代码：
package cn.campusshop.controller;
import java.io.IOException;
import java.util.List;
import javax.servlet.ServletException;
import javax.servlet.annotation.WebServlet;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.http.ParseException;
import org.apache.http.client.HttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import com.alibaba.fastjson.JSON;
import cn.campusshop.model.HotModel;
import cn.campusshop.utils.URLHandle;
/**
* Servlet implementation class HotController
*/
@WebServlet("/hot")
public class HotController extends HttpServlet {
private static final long serialVersionUID = 1L;
/**
* Default constructor.
*/
public HotController() {
// TODO Auto-generated constructor stub
}
/**
* @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse response)
*/
protected void doGet(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
System.out.println("正在生成客户端...");
HttpClient client = HttpClientBuilder.create().build();
System.out.println("客户端生成完毕.");
String url = "https://s.weibo.com/top/summary";
List hotList = null;
// 开始解析
try {
System.out.println("开始响应客户端...");
hotList = URLHandle.urlParser(client, url);
System.out.println("响应完成.");
} catch (ParseException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
System.out.println("开始输出结果...");
for (HotModel hot : hotList) {
System.out.println(hot);
}
response.setContentType("application/json;charset=utf-8");
String string = JSON.toJSONString(hotList);
// string = new String(string.getBytes("ISO-8859-1"), "UTF-8");
response.setHeader("Access-Control-Allow-Origin", "*");
response.getWriter().write(string);
System.out.println("整个结果输出完毕，程序结束.");
}
/**
* @see HttpServlet#doPost(HttpServletRequest request, HttpServletResponse response)
*/
protected void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
// TODO Auto-generated method stub
doGet(request, response);
}
}
HotModel代码：
/**
* Copyright (C) 2019 Baidu, Inc. All Rights Reserved.
*/
package cn.campusshop.model;
/**
* @author zhulang
* @version 创建时间：2019年9月5日 下午11:26:41
*/
public class HotModel {
private String hotRank;// 热搜排名
private String hotName;// 热搜名
private String releaseTime;// 热搜度
/**
* @return the hotRank
*/
public String getHotRank() {
return hotRank;
}
/**
* @param hotRank the hotRank to set
*/
public void setHotRank(String hotRank) {
this.hotRank = hotRank;
}
/**
* @return the hotName
*/
public String getHotName() {
return hotName;
}
/**
* @param hotName the hotName to set
*/
public void setHotName(String hotName) {
this.hotName = hotName;
}
/**
* @return the releaseTime
*/
public String getReleaseTime() {
return releaseTime;
}
/**
* @param releaseTime the releaseTime to set
*/
public void setReleaseTime(String releaseTime) {
this.releaseTime = releaseTime;
}
/*
* (non-Javadoc)
*
* @see java.lang.Object#toString()
*/
@Override
public String toString() {
return "HotModel [hotRank=" + hotRank + ", hotName=" + hotName + ", releaseTime=" + releaseTime + "]";
}
}
URLHandle代码：
/**
* Copyright (C) 2019 Baidu, Inc. All Rights Reserved.
*/
package cn.campusshop.utils;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.util.EntityUtils;
import cn.campusshop.model.HotModel;
/**
* @author zhulang
* @version 创建时间：2019年9月5日 下午11:40:21
*/
public class URLHandle {
public static List urlParser(HttpClient client, String url) throws IOException {
// 创建一个接受数据的数组
List data = new ArrayList<>();
// 获取响应资源
HttpResponse response = HTTPUtils.getHtml(client, url);
// 获取响应状态码
int statusCode = response.getStatusLine().getStatusCode();
System.out.println(statusCode);
if (statusCode == 200) {// 200表示成功
// 获取响应实体内容，并且将其转换为utf-8形式的字符串编码
String entity = EntityUtils.toString(response.getEntity(), "utf-8");
System.out.println("开始解析...");
data = HotParse.getData(entity);
System.out.println("URL解析完成.");
} else {
EntityUtils.consume(response.getEntity());// 释放资源实体
}
System.out.println("返回数据.");
return data;
}
}
HotParse代码：
/**
* Copyright (C) 2019 Baidu, Inc. All Rights Reserved.
*/
package cn.campusshop.utils;
import java.util.ArrayList;
import java.util.List;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import cn.campusshop.model.HotModel;
/**
* @author zhulang
* @version 创建时间：2019年9月5日 下午11:42:29
*/
public class HotParse {
public static List getData(String entity) {
List data = new ArrayList<>();
// 采用jsoup解析
Document doc = Jsoup.parse(entity);
// 根据页面内容分析出需要的元素
Elements elements = doc.select("tbody").select("tr");
System.out.println(elements);
for (Element element : elements) {
HotModel hot = new HotModel();
hot.setHotRank(element.select(".td-01").text());// class等于board-index的i标签
hot.setHotName(element.select("td[class=\"td-02\"] a").text());// 带有class属性的p元素
hot.setReleaseTime(element.select("td[class=\"td-02\"] span").text());
data.add(hot);
}
return data;
}
}
HTTPUtils代码：
/**
* Copyright (C) 2019 Baidu, Inc. All Rights Reserved.
*/
package cn.campusshop.utils;
import java.io.IOException;
/**
* @author zhulang
*@version 创建时间：2019年9月5日 下午11:44:39
*/
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.HttpVersion;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.message.BasicHttpResponse;
public class HTTPUtils {
public static HttpResponse getHtml(HttpClient client, String url) {
// 获取响应文件，即HTML，采用get方法获取响应数据
HttpGet getMethod = new HttpGet(url);
HttpResponse response = new BasicHttpResponse(HttpVersion.HTTP_1_1, HttpStatus.SC_OK, "OK");
try {
// 通过client执行get方法
response = client.execute(getMethod);
} catch (IOException e) {
e.printStackTrace();
} finally {
// getMethod.abort();
}
return response;
}
}
pom文件jar包
4.0.0
cn.campusshop
realtimehot
0.0.1-SNAPSHOT
war
org.jsoup
jsoup
1.11.3
org.apache.httpcomponents
httpclient
4.5.8
org.apache.httpcomponents
httpcore
4.4.11
commons-logging
commons-logging
1.2
commons-codec
commons-codec
1.11
javax.servlet
javax.servlet-api
3.1.0
provided
com.alibaba
fastjson
1.2.6
junit
junit
4.12
org.apache.maven.plugins
maven-compiler-plugin
1.8
1.8
utf-8
org.apache.tomcat.maven
tomcat7-maven-plugin
2.2
8080
utf-8
true
/
tomcat8
http://localhost:8080/manager/text
跨域
* e.preventDefault();bootstrapvalidator组织浏览器默认提交
*var formdata=$("#form").serialize();//用来获取表单中的数据
//跨域：协议+IP地址+端口号，只要有一样不同就涉及到跨域，也就是不同服务器端资源的交互。
// 跨域的解决方法一：在服务器端设置response.setHeader("Access-Control-Allow-Origin", "http://localhost:63342");
// 跨域的解决方法二：在客户端设置响应数据类型dataType:"jsonp",
//jsonp用法：服务器端响应数据时需要将响应的json数据包裹在callback()里面；例如： response.getWriter().write(callback + "(" + json + ")");