Spark Streaming 实战 日志分析（二）数据可视化
1 需求
使用echarts可视化工具将之前统计好的数据进行展示。
2 开发环境
- IDEA+maven
- spring boot + ECharts
3 最终效果
4 编程
代码地址
1）pom.xml，添加依赖
<repositories>
<repository>
<id>clouderaid>
<url>https://repository.cloudera.com/artifactory/cloudera-reposurl>
repository>
repositories>
<dependency>
<groupId>org.springframework.bootgroupId>
<artifactId>spring-boot-starter-thymeleafartifactId>
dependency>
<dependency>
<groupId>org.codehaus.groovygroupId>
<artifactId>groovyartifactId>
<version>2.5.0version>
dependency>
<dependency>
<groupId>org.apache.hbasegroupId>
<artifactId>hbase-clientartifactId>
<version>${hbase.version}version>
dependency>
<dependency>
<groupId>net.sf.json-libgroupId>
<artifactId>json-libartifactId>
<version>2.4version>
<classifier>jdk15classifier>
dependency>2）echarts.html，展示页面
<html lang="en">
<head>
<meta charset="UTF-8"/>
<title>imooc_stattitle>
<script src="js/echarts.min.js">script>
<script src="js/jquery.js">script>
head>
<body>
<div id="main" style="width: 600px;height:400px;position: absolute; top:50%; left: 50%; margin-top: -200px;margin-left: -300px">div>
<script type="text/javascript">
// 基于准备好的dom，初始化echarts实例
var myChart = echarts.init(document.getElementById('main'));
// 指定图表的配置项和数据
var option = {
title : {
text: '慕课网实战课程实时访问量统计',
subtext: '实战课程访问次数',
x:'center'
},
tooltip : {
trigger: 'item',
formatter: "{a}
{b} : {c} ({d}%)"
},
// legend: {
// orient: 'vertical',
// left: 'left'
// },
series : [
{
name: '访问次数',
type: 'pie',
radius : '55%',
center: ['50%', '60%'],
data: (function(){ //
var datas = [];
$.ajax({
type: "POST",
url: "/lihaogn/course_clickcount",
dataType: 'json',
async: false,
success: function(result) {
for(var i=0; i"value" :result[i].value, "name":result[i].name})
}
}
})
return datas;
//]]>
})(),
itemStyle: {
emphasis: {
shadowBlur: 10,
shadowOffsetX: 0,
shadowColor: 'rgba(0, 0, 0, 0.5)'
}
}
}
]
};
// 使用刚指定的配置项和数据显示图表。
myChart.setOption(option);
script>
body>
html>3）HBaseUtils.java，操作数据库
package com.lihaogn.spark.web.com.lihaogn.spark.web.utils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.PrefixFilter;
import org.apache.hadoop.hbase.util.Bytes;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
/**
* HBase操作工具类
*/
public class HBaseUtils {
HBaseAdmin admin = null;
Configuration conf = null;
/**
* 私有构造方法：加载一些必要的参数
*/
private HBaseUtils() {
conf = new Configuration();
conf.set("hbase.zookeeper.quorum", "localhost:2181");
conf.set("hbase.rootdir", "hdfs://localhost:8020/hbase");
try {
admin = new HBaseAdmin(conf);
} catch (IOException e) {
e.printStackTrace();
}
}
private static HBaseUtils instance = null;
public static synchronized HBaseUtils getInstance() {
if (null == instance) {
instance = new HBaseUtils();
}
return instance;
}
/**
* 根据表名获取到HTable实例
*/
public HTable getTable(String tableName) {
HTable table = null;
try {
table = new HTable(conf, tableName);
} catch (IOException e) {
e.printStackTrace();
}
return table;
}
/**
* 根据表名和输入条件获取HBase的记录数
*/
public Map query(String tableName, String condition) throws Exception {
Map map = new HashMap();
HTable table = getTable(tableName);
String cf = "info";
String qualifier = "click_count";
Scan scan = new Scan();
Filter filter = new PrefixFilter(Bytes.toBytes(condition));
scan.setFilter(filter);
ResultScanner rs = table.getScanner(scan);
for (Result result : rs) {
String row = Bytes.toString(result.getRow());
long clickCount = Bytes.toLong(result.getValue(cf.getBytes(),qualifier.getBytes()));
map.put(row, clickCount);
}
return map;
}
public static void main(String[] args) throws Exception {
// HTable table = HBaseUtils.getInstance().getTable("course_clickcount");
// System.out.println(table.getName().getNameAsString());
Map map = HBaseUtils.getInstance().query("course_clickcount", "20180906_");
for (Map.Entry entry : map.entrySet()) {
System.out.println(entry.getKey() + ":" + entry.getValue());
}
}
}
4）CourseClickCount.java，实体类
package com.lihaogn.spark.web.com.lihaogn.spark.web.domain;
import org.springframework.stereotype.Component;
/**
* 课程点击数实体类
*/
@Component
public class CourseClickCount {
private String name;
private long value;
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public long getValue() {
return value;
}
public void setValue(long value) {
this.value = value;
}
}
5）CourseClickCountDAO.java，访问数据库
package com.lihaogn.spark.web.com.lihaogn.spark.web.dao;
import com.lihaogn.spark.web.com.lihaogn.spark.web.domain.CourseClickCount;
import com.lihaogn.spark.web.com.lihaogn.spark.web.utils.HBaseUtils;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/**
* 课程访问数量数据访问
*/
@Component
public class CourseClickCountDAO {
/**
* 根据日期查询
*
* @param day
* @return
* @throws Exception
*/
public List query(String day) throws Exception {
List list = new ArrayList<>();
// 去HBase表中根据day获取课程访问量
Map map = HBaseUtils.getInstance().query("course_clickcount", "20180906");
for (Map.Entry entry : map.entrySet()) {
CourseClickCount model = new CourseClickCount();
model.setName(entry.getKey());
model.setValue(entry.getValue());
list.add(model);
}
return list;
}
public static void main(String[] args) throws Exception {
CourseClickCountDAO dao = new CourseClickCountDAO();
List list = dao.query("20180906");
for (CourseClickCount model : list) {
System.out.println(model.getName() + " : " + model.getValue());
}
}
}
6）SparkStatApp.java，web后端
package com.lihaogn.spark.web.com.lihaogn.spark.web.spark;
import com.lihaogn.spark.web.com.lihaogn.spark.web.dao.CourseClickCountDAO;
import com.lihaogn.spark.web.com.lihaogn.spark.web.domain.CourseClickCount;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.servlet.ModelAndView;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* web
*/
@RestController
public class SparkStatApp {
private static Map courses = new HashMap<>();
static {
courses.put("112", "Spark SQL慕课网日志分析");
courses.put("128", "10小时入门大数据");
courses.put("145", "深度学习之神经网络核心原理与算法");
courses.put("146", "强大的Node.js在Web开发的应用");
courses.put("131", "Vue+Django实战");
courses.put("130", "Web前端性能优化");
}
@Autowired
CourseClickCountDAO courseClickCountDAO;
@RequestMapping(value = "/course_clickcount",method = RequestMethod.POST)
@ResponseBody
public List courseClickCount() throws Exception {
List list = courseClickCountDAO.query("20180906");
for (CourseClickCount model : list) {
model.setName(courses.get(model.getName().substring(9)));
}
return list;
}
@RequestMapping(value = "/echarts", method = RequestMethod.GET)
public ModelAndView echarts() {
return new ModelAndView("echarts");
}
}
7）WebApplication.java，主程序
package com.lihaogn.spark.web;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@SpringBootApplication
public class WebApplication {
public static void main(String[] args) {
SpringApplication.run(WebApplication.class, args);
}
}