第1章 项目需求及架构设计
1.1 需求
- 数据采集平台搭建
- Kafka、Zookeeper中间件准备
- 下游Spark Streaming对接Kafka接收数据，
- 实现vip个数统计
- 栏目打标签功能
- 做题正确率与掌握度的实时计算功能。
1.2 项目框架
1.2.1 技术选型
一、数据存储：Kafka、MySql
二、数据处理：Spark
三、其他组件：Zookeeper
1.2.2 流程设计
流程图
1.3 思考
（1）Spark Streaming 下每个stage的耗时由什么决定
（2）Spark Streaming task发生数据倾斜如何解决
（3）Spark Streaming操作mysql时，相同维度的数据如何保证线程安全问题
（4）如何保证kill Spark Streaming任务的时候不丢失数据
（5）如何保证Spark Streaming的第一次启动和kill后第二次启动时据不丢失数据
（6）Spark Streaming下如何正确操作mysql（如何正确使用连接）
（7）MySql建表时 索引注意
pom.xml文件
education-online
com.qingfeng
1.0-SNAPSHOT
4.0.0
com_qingfeng_warehouse
org.apache.spark
spark-core_2.11
${spark.version}
org.apache.spark
spark-sql_2.11
${spark.version}
org.apache.spark
spark-hive_2.11
${spark.version}
org.scala-lang
scala-library
com.alibaba
fastjson
1.2.47
mysql
mysql-connector-java
5.1.46
org.scala-tools
maven-scala-plugin
2.15.1
compile-scala
add-source
compile
test-compile-scala
add-source
testCompile
org.apache.maven.plugins
maven-assembly-plugin
jar-with-dependencies
MySQL配置文件
jdbc.url=jdbc:mysql://hadoop102:3306/course_learn?useUnicode=true&characterEncoding=utf8&serverTimezone=Asia/Shanghai&useSSL=false
jdbc.user=root
jdbc.password=000000
读取配置文件工具类
import java.io.InputStream;
import java.util.Properties;
/**
*
* 读取配置文件工具类
*/
public class ConfigurationManager {
private static Properties prop = new Properties();
static {
try {
InputStream inputStream = ConfigurationManager.class.getClassLoader()
.getResourceAsStream("comerce.properties"); //自动加载配置文件
prop.load(inputStream);
} catch (Exception e) {
e.printStackTrace();
}
}
//获取配置项
public static String getProperty(String key) {
return prop.getProperty(key);
}
//获取布尔类型的配置项
public static boolean getBoolean(String key) {
String value = prop.getProperty(key);
try {
return Boolean.valueOf(value); //解析布尔值
} catch (Exception e) {
e.printStackTrace();
}
return false;
}
}
Json解析工具类
package com.atguigu.qzpoint.util;
import com.alibaba.fastjson.JSONObject;
public class ParseJsonData {
public static JSONObject getJsonData(String data) {
try {
return JSONObject.parseObject(data); Java代码
} catch (Exception e) {
return null;
}
}
}
Druid连接池
import com.alibaba.druid.pool.DruidDataSourceFactory;
import javax.sql.DataSource;
import java.io.Serializable;
import java.sql.*;
import java.util.Properties;
/**
* 德鲁伊连接池
*/
public class DataSourceUtil implements Serializable {
public static DataSource dataSource = null;
static { //数据库连接池放在静态代码块中，保证只创建一份
try {
Properties props = new Properties();
props.setProperty("url", ConfigurationManager.getProperty("jdbc.url"));
props.setProperty("username", ConfigurationManager.getProperty("jdbc.user"));
props.setProperty("password", ConfigurationManager.getProperty("jdbc.password"));
props.setProperty("initialSize", "5"); //初始化大小
props.setProperty("maxActive", "10"); //最大连接
props.setProperty("minIdle", "5"); //最小连接
props.setProperty("maxWait", "60000"); //等待时长
props.setProperty("timeBetweenEvictionRunsMillis", "2000");//配置多久进行一次检测,检测需要关闭的连接 单位毫秒
props.setProperty("minEvictableIdleTimeMillis", "600000");//配置连接在连接池中最小生存时间 单位毫秒
props.setProperty("maxEvictableIdleTimeMillis", "900000"); //配置连接在连接池中最大生存时间 单位毫秒
props.setProperty("validationQuery", "select 1");
props.setProperty("testWhileIdle", "true");
props.setProperty("testOnBorrow", "false");
props.setProperty("testOnReturn", "false");
props.setProperty("keepAlive", "true");
props.setProperty("phyMaxUseCount", "100000");
// props.setProperty("driverClassName", "com.mysql.jdbc.Driver");
dataSource = DruidDataSourceFactory.createDataSource(props); 连接池只有一份
} catch (Exception e) {
e.printStackTrace();
}
}
//提供获取连接的方法
public static Connection getConnection() throws SQLException {
return dataSource.getConnection();
}
// 提供关闭资源的方法【connection是归还到连接池】
// 提供关闭资源的方法 【方法重载】3 dql
public static void closeResource(ResultSet resultSet, PreparedStatement preparedStatement,
Connection connection) {
// 关闭结果集
// ctrl+alt+m 将java语句抽取成方法
closeResultSet(resultSet);
// 关闭语句执行者
closePrepareStatement(preparedStatement);
// 关闭连接
closeConnection(connection);
}
private static void closeConnection(Connection connection) {
if (connection != null) { //防止空指针异常
try {
connection.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
}
private static void closePrepareStatement(PreparedStatement preparedStatement) {
if (preparedStatement != null) {
try {
preparedStatement.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
}
private static void closeResultSet(ResultSet resultSet) {
if (resultSet != null) {
try {
resultSet.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
}
}
创建MySQL代理类
import java.sql.{Connection, PreparedStatement, ResultSet}
trait QueryCallback {
def process(rs: ResultSet)
}
class SqlProxy {
private var rs: ResultSet = _
private var psmt: PreparedStatement = _
/**
* 执行修改语句
*
* @param conn
* @param sql
* @param params
* @return
*/
def executeUpdate(conn: Connection, sql: String, params: Array[Any]): Int = {
var rtn = 0
try {
psmt = conn.prepareStatement(sql)
if (params != null && params.length > 0) { //注意防止空指针异常
for (i <- 0 until params.length) {
psmt.setObject(i + 1, params(i))
}
}
rtn = psmt.executeUpdate()
} catch {
case e: Exception => e.printStackTrace()
}
rtn //状态码？
}
/**
* 执行查询语句
* @param conn
* @param sql
* @param params
* @return
*/
def executeQuery(conn: Connection, sql: String, params: Array[Any], queryCallback: QueryCallback) = {
rs = null
try {
psmt = conn.prepareStatement(sql)
if (params != null && params.length > 0) {
for (i <- 0 until params.length) {
psmt.setObject(i + 1, params(i))
}
}
rs = psmt.executeQuery()
queryCallback.process(rs)
} catch {
case e: Exception => e.printStackTrace()
}
}
def shutdown(conn: Connection): Unit = DataSourceUtil.closeResource(rs, psmt, conn)
}
指标一：实时统计注册人数代码实现
import java.lang
import java.sql.ResultSet
import java.util.Random
import com.atguigu.qzpoint.util.{DataSourceUtil, QueryCallback, SqlProxy}
import org.apache.kafka.clients.consumer.ConsumerRecord
import org.apache.kafka.common.TopicPartition
import org.apache.kafka.common.serialization.StringDeserializer
import org.apache.spark.SparkConf
import org.apache.spark.streaming.dstream.InputDStream
import org.apache.spark.streaming.kafka010._
import org.apache.spark.streaming.{Seconds, StreamingContext}
import scala.collection.mutable
object RegisterStreaming {
private val groupid = "register_group_test"
def main(args: Array[String]): Unit = {
val conf = new SparkConf().setAppName(this.getClass.getSimpleName).setMaster("local[2]")
.set("spark.streaming.kafka.maxRatePerPartition", "50") //设置每秒每个分区拉取信息最大条数
.set("spark.streaming.stopGracefullyOnShutdown", "true") //开启优雅的关闭
// 还应该开启背压机制（削峰）
val ssc = new StreamingContext(conf, Seconds(3))
val topics = Array("register_topic")
val kafkaMap: Map[String, Object] = Map[String, Object](
"bootstrap.servers" -> "hadoop102:9092,hadoop103:9092,hadoop104:9092",
"key.deserializer" -> classOf[StringDeserializer],
"value.deserializer" -> classOf[StringDeserializer],
"group.id" -> groupid,
"auto.offset.reset" -> "earliest",
"enable.auto.commit" -> (false: lang.Boolean)
)
//设置checkpoint路径
ssc.checkpoint("hdfs://hadoop102:9000/user/qingfeng/sparkstreaming/checkpoint")
//查询mysql中是否有偏移量
val sqlProxy = new SqlProxy()
val offsetMap = new mutable.HashMap[TopicPartition, Long]()
val client = DataSourceUtil.getConnection
try {
sqlProxy.executeQuery(client, "select * from `offset_manager` where groupid=?", Array(groupid), new QueryCallback {
override def process(rs: ResultSet): Unit = {
while (rs.next()) {
val model = new TopicPartition(rs.getString(2), rs.getInt(3)) //获取topic、partition
val offset = rs.getLong(4) //获取偏移量
offsetMap.put(model, offset)
}
rs.close() //关闭游标 }
})
} catch {
case e: Exception => e.printStackTrace()
} finally {
sqlProxy.shutdown(client)
}
//设置kafka消费数据的参数 判断本地是否有偏移量 有则根据偏移量继续消费 无则重新消费
val stream: InputDStream[ConsumerRecord[String, String]] = if (offsetMap.isEmpty) {
KafkaUtils.createDirectStream( ssc, LocationStrategies.PreferConsistent, ConsumerStrategies.Subscribe[String, String](topics, kafkaMap))
} else {
KafkaUtils.createDirectStream(ssc, LocationStrategies.PreferConsistent, ConsumerStrategies.Subscribe[String, String](topics, kafkaMap, offsetMap))
}
//对数据先进行过滤，后处理
val resultDStream = stream.filter(item => item.value().split("\t").length == 3). mapPartitions(partitions => {
partitions.map(item => {
val line = item.value()
val arr = line.split("\t")
val app_name = arr(1) match {
case "1" => "PC"
case "2" => "APP"
case _ => "Other"
}
(app_name, 1)
})
})
resultDStream.cache()
// resultDStream.reduceByKeyAndWindow((x: Int, y: Int) => x + y, Seconds(60), Seconds(6)).print()
val updateFunc = (values: Seq[Int], state: Option[Int]) => {
val currentCount = values.sum //本批次求和
val previousCount = state.getOrElse(0) //历史数据
Some(currentCount + previousCount)
}
resultDStream.updateStateByKey(updateFunc).print()
// val dsStream = stream.filter(item => item.value().split("\t").length == 3)
// .mapPartitions(partitions =>
// partitions.map(item => {
// **val rand = new Random()**
// val line = item.value()
// val arr = line.split("\t")
// val app_id = arr(1)
// (rand.nextInt(3) + "_" + app_id, 1) //在key中添加随机数，防止数据倾斜 }))
// dsStream.print()
// val a = dsStream.reduceByKey(_ + _)
// a.print()
// a.map(item => {
// val appid = item._1.split("_")(1)
// (appid, item._2)
// }).reduceByKey(_ + _).print()
//处理完 业务逻辑后 手动提交offset维护到本地 mysql中
stream.foreachRDD(rdd => {
val sqlProxy = new SqlProxy()
val client = DataSourceUtil.getConnection
try {
val offsetRanges: Array[OffsetRange] = **rdd**.**asInstanceOf[HasOffsetRanges].offsetRanges**
for (or <- offsetRanges) {
sqlProxy.executeUpdate(client, "replace into `offset_manager` (groupid,topic,`partition`,untilOffset) values(?,?,?,?)",Array(groupid, or.topic, or.partition.toString, or.untilOffset))
}
} catch {
case e: Exception => e.printStackTrace()
} finally {
sqlProxy.shutdown(client)
}
})
ssc.start()
ssc.awaitTermination()
}
}
offset表结构：
| groupid | topic | partition | untiloffset |
|---|---|---|---|
| groupid01 | topic01 | 1 | 12123 |
MySQL中replace into用法详解
replace into t(id, update_time) values(1, now());
或
replace into t(id, update_time) select 1, now();
replace into 跟 insert 功能类似，不同点在于：replace into 首先尝试插入数据到表中，
1. 如果发现表中已经有此行数据（根据主键或者唯一索引判断）则先删除此行数据，然后插入新的数据。
2. 否则，直接插入新数据。
要注意的是：插入数据的表必须有主键或者是唯一索引！否则的话，replace into 会直接插入数据，这将导致表中出现重复的数据。
MySQL replace into 有三种形式：
1. replace into tbl_name(col_name, ...) values(...)
2. replace into tbl_name(col_name, ...) select ...
3. replace into tbl_name set col_name=value, ...
第一种形式类似于insert into的用法，
第二种replace select的用法也类似于insert select，这种用法并不一定要求列名匹配，事实上，MYSQL甚至不关心select返回的列名，它需要的是列的位置。
例如，replace into tb1( name, title, mood) select rname, rtitle, rmood from tb2;这个例子使用replace into从tb2中将所有数据导入tb1中。
第三种replace set用法类似于update set用法，使用一个例如“SET col_name = col_name + 1”的赋值，则对位于右侧的列名称的引用会被作为DEFAULT(col_name)处理。因此，该赋值相当于SET col_name = DEFAULT(col_name) + 1。
前两种形式用的多些。其中 “into” 关键字可以省略，不过最好加上 “into”，这样意思更加直观。另外，对于那些没有给予值的列，MySQL 将自动为这些列赋上默认值。
指标二：实时统计学员做题正确率与知识点掌握度
import java.lang
import java.sql.{Connection, ResultSet}
import java.time.LocalDateTime
import java.time.format.DateTimeFormatter
import com.atguigu.qzpoint.util.{DataSourceUtil, QueryCallback, SqlProxy}
import org.apache.kafka.clients.consumer.ConsumerRecord
import org.apache.kafka.common.TopicPartition
import org.apache.kafka.common.serialization.StringDeserializer
import org.apache.spark.SparkConf
import org.apache.spark.streaming.dstream.InputDStream
import org.apache.spark.streaming.kafka010._
import org.apache.spark.streaming.{Seconds, StreamingContext}
import scala.collection.mutable
/**
* 知识点掌握度实时统计
*/
object QzPointStreaming {
private val groupid = "qz_point_group"
def main(args: Array[String]): Unit = {
val conf = new SparkConf().setAppName(this.getClass.getSimpleName).setMaster("local[*]")
.set("spark.streaming.kafka.maxRatePerPartition", "50")
.set("spark.streaming.stopGracefullyOnShutdown", "true")
val ssc = new StreamingContext(conf, Seconds(3))
val topics = Array("qz_log")
val kafkaMap: Map[String, Object] = Map[String, Object](
"bootstrap.servers" -> "hadoop102:9092,hadoop103:9092,hadoop104:9092",
"key.deserializer" -> classOf[StringDeserializer],
"value.deserializer" -> classOf[StringDeserializer],
"group.id" -> groupid,
"auto.offset.reset" -> "earliest", //保证第一次消费不丢数据
"enable.auto.commit" -> (false: lang.Boolean)
)
//查询mysql中是否存在偏移量
val sqlProxy = new SqlProxy()
val offsetMap = new mutable.HashMap[TopicPartition, Long]()
val client = DataSourceUtil.getConnection
try {
sqlProxy.executeQuery(client, "select * from `offset_manager` where groupid=?", Array(groupid), new QueryCallback {
override def process(rs: ResultSet): Unit = {
while (rs.next()) {
val model = new TopicPartition(rs.getString(2), rs.getInt(3))
val offset = rs.getLong(4)
offsetMap.put(model, offset)
}
rs.close() //关闭游标
}
})
} catch {
case e: Exception => e.printStackTrace()
} finally {
sqlProxy.shutdown(client)
}
//设置kafka消费数据的参数 判断本地是否有偏移量 有则根据偏移量继续消费 无则重新消费
val stream: InputDStream[ConsumerRecord[String, String]] = if (offsetMap.isEmpty) {
KafkaUtils.createDirectStream(
ssc, LocationStrategies.PreferConsistent, ConsumerStrategies.Subscribe[String, String](topics, kafkaMap))
} else {
KafkaUtils.createDirectStream(
ssc, LocationStrategies.PreferConsistent, ConsumerStrategies.Subscribe[String, String](topics, kafkaMap, offsetMap))
}
//过滤不正常数据 获取数据
val dsStream = stream.filter(item => item.value().split("\t").length == 6).
mapPartitions(partition => partition.map(item => {
val line = item.value()
val arr = line.split("\t")
val uid = arr(0) //用户id
val courseid = arr(1) //课程id
val pointid = arr(2) //知识点id
val questionid = arr(3) //题目id
val istrue = arr(4) //是否正确
val createtime = arr(5) //创建时间
(uid, courseid, pointid, questionid, istrue, createtime)
}))
dsStream.foreachRDD(rdd => {
//获取相同用户 同一课程 同一知识点的数据
val groupRdd = rdd.groupBy(item => item._1 + "-" + item._2 + "-" + item._3)
groupRdd.foreachPartition(partition => {
//在分区下获取jdbc连接
val sqlProxy = new SqlProxy()
val client = DataSourceUtil.getConnection
try {
partition.foreach { case (key, iters) =>
qzQuestionUpdate(key, iters, sqlProxy, client) //对题库进行更新操作
}
} catch {
case e: Exception => e.printStackTrace()
}
finally {
sqlProxy.shutdown(client)
}
}
)
})
//处理完 业务逻辑后 手动提交offset维护到本地 mysql中
stream.foreachRDD(rdd => {
val sqlProxy = new SqlProxy()
val client = DataSourceUtil.getConnection
try {
val offsetRanges: Array[OffsetRange] = rdd.asInstanceOf[HasOffsetRanges].offsetRanges
for (or <- offsetRanges) {
sqlProxy.executeUpdate(client, "replace into `offset_manager` (groupid,topic,`partition`,untilOffset) values(?,?,?,?)",
Array(groupid, or.topic, or.partition.toString, or.untilOffset))
}
} catch {
case e: Exception => e.printStackTrace()
} finally {
sqlProxy.shutdown(client)
}
})
ssc.start()
ssc.awaitTermination()
}
/**
* 对题目表进行更新操作
*
* @param key
* @param iters
* @param sqlProxy
* @param client
* @return
*/
def qzQuestionUpdate(key: String, iters: Iterable[(String, String, String, String, String, String)], sqlProxy: SqlProxy, client: Connection) = {
val keys = key.split("-")
val userid = keys(0).toInt
val courseid = keys(1).toInt
val pointid = keys(2).toInt
val array = iters.toArray
val questionids = array.map(_._4).distinct //对当前批次的数据下questionid 去重
//查询历史数据下的 questionid
var questionids_history: Array[String] = Array()
sqlProxy.executeQuery(client, "select questionids from qz_point_history where userid=? and courseid=? and pointid=?",
Array(userid, courseid, pointid), new QueryCallback {
override def process(rs: ResultSet): Unit = {
while (rs.next()) {
questionids_history = rs.getString(1).split(",")
}
rs.close() //关闭游标
}
})
//获取到历史数据后再与当前数据进行拼接 去重
val resultQuestionid = questionids.union(questionids_history).distinct
val countSize = resultQuestionid.length
val resultQuestionid_str = resultQuestionid.mkString(",")
val qz_count = questionids.length //去重后的题个数
var qz_sum = array.length //获取当前批次题总数
var qz_istrue = array.filter(_._5.equals("1")).size //获取当前批次做正确的题个数
val createtime = array.map(_._6).min //获取最早的创建时间 作为表中创建时间
//更新qz_point_set 记录表 此表用于存当前用户做过的questionid表
val updatetime = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss").format(LocalDateTime.now())
sqlProxy.executeUpdate(client, "insert into qz_point_history(userid,courseid,pointid,questionids,createtime,updatetime) values(?,?,?,?,?,?) " +
" on duplicate key update questionids=?,updatetime=?", Array(userid, courseid, pointid, resultQuestionid_str, createtime, createtime, resultQuestionid_str, updatetime))
var qzSum_history = 0
var istrue_history = 0
sqlProxy.executeQuery(client, "select qz_sum,qz_istrue from qz_point_detail where userid=? and courseid=? and pointid=?",
Array(userid, courseid, pointid), new QueryCallback {
override def process(rs: ResultSet): Unit = {
while (rs.next()) {
qzSum_history += rs.getInt(1)
istrue_history += rs.getInt(2)
}
rs.close()
}
})
qz_sum += qzSum_history
qz_istrue += istrue_history
val correct_rate = qz_istrue.toDouble / qz_sum.toDouble //计算正确率
//计算完成率
//假设每个知识点下一共有30道题 先计算题的做题情况 再计知识点掌握度
val qz_detail_rate = countSize.toDouble / 30 //算出做题情况乘以 正确率 得出完成率 假如30道题都做了那么正确率等于 知识点掌握度
val mastery_rate = qz_detail_rate * correct_rate
sqlProxy.executeUpdate(client, "insert into qz_point_detail(userid,courseid,pointid,qz_sum,qz_count,qz_istrue,correct_rate,mastery_rate,createtime,updatetime)" +
" values(?,?,?,?,?,?,?,?,?,?) on duplicate key update qz_sum=?,qz_count=?,qz_istrue=?,correct_rate=?,mastery_rate=?,updatetime=?",
Array(userid, courseid, pointid, qz_sum, countSize, qz_istrue, correct_rate, mastery_rate, createtime, updatetime, qz_sum, countSize, qz_istrue, correct_rate, mastery_rate, updatetime))
}
}
指标三：实时统计商品页到订单页，订单页到支付页转换率
import java.lang
import java.sql.{Connection, ResultSet}
import java.text.NumberFormat
import com.atguigu.qzpoint.util.{DataSourceUtil, ParseJsonData, QueryCallback, SqlProxy}
import org.apache.kafka.clients.consumer.ConsumerRecord
import org.apache.kafka.common.TopicPartition
import org.apache.kafka.common.serialization.StringDeserializer
import org.apache.spark.streaming.dstream.InputDStream
import org.apache.spark.streaming.kafka010._
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.{SparkConf, SparkFiles}
import org.lionsoul.ip2region.{DbConfig, DbSearcher}
import scala.collection.mutable
import scala.collection.mutable.ArrayBuffer
/**
* 页面转换率实时统计
*/
object PageStreaming {
private val groupid = "vip_count_groupid"
def main(args: Array[String]): Unit = {
val conf = new SparkConf().setAppName(this.getClass.getSimpleName).setMaster("local[*]")
.set("spark.streaming.kafka.maxRatePerPartition", "30")
.set("spark.streaming.stopGracefullyOnShutdown", "true")
val ssc = new StreamingContext(conf, Seconds(3))
val topics = Array("page_topic")
val kafkaMap: Map[String, Object] = Map[String, Object](
"bootstrap.servers" -> "hadoop102:9092,hadoop103:9092,hadoop104:9092",
"key.deserializer" -> classOf[StringDeserializer],
"value.deserializer" -> classOf[StringDeserializer],
"group.id" -> groupid,
"auto.offset.reset" -> "earliest",
"enable.auto.commit" -> (false: lang.Boolean)
)
//查询mysql中是否存在偏移量
val sqlProxy = new SqlProxy()
val offsetMap = new mutable.HashMap[TopicPartition, Long]()
val client = DataSourceUtil.getConnection
try {
sqlProxy.executeQuery(client, "select *from `offset_manager` where groupid=?", Array(groupid), new QueryCallback {
override def process(rs: ResultSet): Unit = {
while (rs.next()) {
val model = new TopicPartition(rs.getString(2), rs.getInt(3))
val offset = rs.getLong(4)
offsetMap.put(model, offset)
}
rs.close()
}
})
} catch {
case e: Exception => e.printStackTrace()
} finally {
sqlProxy.shutdown(client)
}
//设置kafka消费数据的参数 判断本地是否有偏移量 有则根据偏移量继续消费 无则重新消费
val stream: InputDStream[ConsumerRecord[String, String]] = if (offsetMap.isEmpty) {
KafkaUtils.createDirectStream(
ssc, LocationStrategies.PreferConsistent, ConsumerStrategies.Subscribe[String, String](topics, kafkaMap))
} else {
KafkaUtils.createDirectStream(
ssc, LocationStrategies.PreferConsistent, ConsumerStrategies.Subscribe[String, String](topics, kafkaMap, offsetMap))
}
//解析json数据
val dsStream = stream.map(item => item.value()).mapPartitions(partition => {
partition.map(item => {
val jsonObject = ParseJsonData.getJsonData(item) //判断是否包含key
val uid = if (jsonObject.containsKey("uid")) jsonObject.getString("uid") else ""
val app_id = if (jsonObject.containsKey("app_id")) jsonObject.getString("app_id") else ""
val device_id = if (jsonObject.containsKey("device_id")) jsonObject.getString("device_id") else ""
val ip = if (jsonObject.containsKey("ip")) jsonObject.getString("ip") else ""
val last_page_id = if (jsonObject.containsKey("last_page_id")) jsonObject.getString("last_page_id") else ""
val pageid = if (jsonObject.containsKey("page_id")) jsonObject.getString("page_id") else ""
val next_page_id = if (jsonObject.containsKey("next_page_id")) jsonObject.getString("next_page_id") else ""
(uid, app_id, device_id, ip, last_page_id, pageid, next_page_id)
})
}).filter(item => {
!item._5.equals("") && !item._6.equals("") && !item._7.equals("")
})
dsStream.cache()
val pageValueDStream = dsStream.map(item => (item._5 + "_" + item._6 + "_" + item._7, 1))
val resultDStream = pageValueDStream.reduceByKey(_ + _)
resultDStream.foreachRDD(rdd => {
rdd.foreachPartition(partition => {
//在分区下获取jdbc连接
val sqlProxy = new SqlProxy()
val client = DataSourceUtil.getConnection
try {
partition.foreach(item => {
calcPageJumpCount(sqlProxy, item, client) //计算页面跳转个数
})
} catch {
case e: Exception => e.printStackTrace()
} finally {
sqlProxy.shutdown(client)
}
})
})
ssc.sparkContext.addFile(this.getClass.getResource("/ip2region.db").getPath) //广播文件
val ipDStream = dsStream.mapPartitions(patitions => {
val dbFile = SparkFiles.get("ip2region.db")
val ipsearch = new DbSearcher(new DbConfig(), dbFile)
patitions.map { item =>
val ip = item._4
val province = ipsearch.memorySearch(ip).getRegion().split("\\|")(2) //获取ip详情 中国|0|上海|上海市|有线通
(province, 1l) //根据省份 统计点击个数
}
}).reduceByKey(_ + _)
ipDStream.foreachRDD(rdd => {
//查询mysql历史数据 转成rdd
val ipSqlProxy = new SqlProxy()
val ipClient = DataSourceUtil.getConnection
try {
val history_data = new ArrayBuffer[(String, Long)]()
ipSqlProxy.executeQuery(ipClient, "select province,num from tmp_city_num_detail", null, new QueryCallback {
override def process(rs: ResultSet): Unit = {
while (rs.next()) {
val tuple = (rs.getString(1), rs.getLong(2))
history_data += tuple
}
}
})
val history_rdd = ssc.sparkContext.makeRDD(history_data)
val resultRdd = history_rdd.fullOuterJoin(rdd).map(item => {
val province = item._1
val nums = item._2._1.getOrElse(0l) + item._2._2.getOrElse(0l)
(province, nums)
})
resultRdd.foreachPartition(partitions => {
val sqlProxy = new SqlProxy()
val client = DataSourceUtil.getConnection
try {
partitions.foreach(item => {
val province = item._1
val num = item._2
//修改mysql数据 并重组返回最新结果数据
sqlProxy.executeUpdate(client, "insert into tmp_city_num_detail(province,num)values(?,?) on duplicate key update num=?",
Array(province, num, num))
})
} catch {
case e: Exception => e.printStackTrace()
} finally {
sqlProxy.shutdown(client)
}
})
val top3Rdd = resultRdd.sortBy[Long](_._2, false).take(3)
sqlProxy.executeUpdate(ipClient, "truncate table top_city_num", null)
top3Rdd.foreach(item => {
sqlProxy.executeUpdate(ipClient, "insert into top_city_num (province,num) values(?,?)", Array(item._1, item._2))
})
} catch {
case e: Exception => e.printStackTrace()
} finally {
sqlProxy.shutdown(ipClient)
}
})
//计算转换率
//处理完 业务逻辑后 手动提交offset维护到本地 mysql中
stream.foreachRDD(rdd => {
val sqlProxy = new SqlProxy()
val client = DataSourceUtil.getConnection
try {
calcJumRate(sqlProxy, client) //计算转换率
val offsetRanges: Array[OffsetRange] = rdd.asInstanceOf[HasOffsetRanges].offsetRanges
for (or <- offsetRanges) {
sqlProxy.executeUpdate(client, "replace into `offset_manager` (groupid,topic,`partition`,untilOffset) values(?,?,?,?)",
Array(groupid, or.topic, or.partition.toString, or.untilOffset))
}
} catch {
case e: Exception => e.printStackTrace()
} finally {
sqlProxy.shutdown(client)
}
})
ssc.start()
ssc.awaitTermination()
}
/**
* 计算页面跳转个数
*
* @param sqlProxy
* @param item
* @param client
*/
def calcPageJumpCount(sqlProxy: SqlProxy, item: (String, Int), client: Connection): Unit = {
val keys = item._1.split("_")
var num: Long = item._2
val page_id = keys(1).toInt //获取当前page_id
val last_page_id = keys(0).toInt //获取上一page_id
val next_page_id = keys(2).toInt //获取下页面page_id
//查询当前page_id的历史num个数
sqlProxy.executeQuery(client, "select num from page_jump_rate where page_id=?", Array(page_id), new QueryCallback {
override def process(rs: ResultSet): Unit = {
while (rs.next()) {
num += rs.getLong(1)
}
rs.close()
}
//对num 进行修改 并且判断当前page_id是否为首页
if (page_id == 1) {
sqlProxy.executeUpdate(client, "insert into page_jump_rate(last_page_id,page_id,next_page_id,num,jump_rate)" +
"values(?,?,?,?,?) on duplicate key update num=num+?", Array(last_page_id, page_id, next_page_id, num, "100%", num))
} else {
sqlProxy.executeUpdate(client, "insert into page_jump_rate(last_page_id,page_id,next_page_id,num)" +
"values(?,?,?,?) on duplicate key update num=num+?", Array(last_page_id, page_id, next_page_id, num, num))
}
})
}
/**
* 计算转换率
*/
def calcJumRate(sqlProxy: SqlProxy, client: Connection): Unit = {
var page1_num = 0l
var page2_num = 0l
var page3_num = 0l
sqlProxy.executeQuery(client, "select num from page_jump_rate where page_id=?", Array(1), new QueryCallback {
override def process(rs: ResultSet): Unit = {
while (rs.next()) {
page1_num = rs.getLong(1)
}
}
})
sqlProxy.executeQuery(client, "select num from page_jump_rate where page_id=?", Array(2), new QueryCallback {
override def process(rs: ResultSet): Unit = {
while (rs.next()) {
page2_num = rs.getLong(1)
}
}
})
sqlProxy.executeQuery(client, "select num from page_jump_rate where page_id=?", Array(3), new QueryCallback {
override def process(rs: ResultSet): Unit = {
while (rs.next()) {
page3_num = rs.getLong(1)
}
}
})
val nf = NumberFormat.getPercentInstance
val page1ToPage2Rate = if (page1_num == 0) "0%" else nf.format(page2_num.toDouble / page1_num.toDouble)
val page2ToPage3Rate = if (page2_num == 0) "0%" else nf.format(page3_num.toDouble / page2_num.toDouble)
sqlProxy.executeUpdate(client, "update page_jump_rate set jump_rate=? where page_id=?", Array(page1ToPage2Rate, 2))
sqlProxy.executeUpdate(client, "update page_jump_rate set jump_rate=? where page_id=?", Array(page2ToPage3Rate, 3))
}
}
指标六：实时统计视频播放各时长
import java.lang
import java.sql.{Connection, ResultSet}
import com.atguigu.qzpoint.bean.LearnModel
import com.atguigu.qzpoint.util.{DataSourceUtil, ParseJsonData, QueryCallback, SqlProxy}
import org.apache.kafka.clients.consumer.ConsumerRecord
import org.apache.kafka.common.TopicPartition
import org.apache.kafka.common.serialization.StringDeserializer
import org.apache.spark.SparkConf
import org.apache.spark.streaming.dstream.InputDStream
import org.apache.spark.streaming.kafka010._
import org.apache.spark.streaming.{Seconds, StreamingContext}
import scala.collection.mutable
import scala.collection.mutable.ArrayBuffer
object CourseLearnStreaming {
private val groupid = "course_learn_test1"
def main(args: Array[String]): Unit = {
val conf = new SparkConf().setAppName(this.getClass.getSimpleName)
.set("spark.streaming.kafka.maxRatePerPartition", "30")
.set("spark.streaming.stopGracefullyOnShutdown", "true")
val ssc = new StreamingContext(conf, Seconds(3))
val topics = Array("course_learn")
val kafkaMap: Map[String, Object] = Map[String, Object](
"bootstrap.servers" -> "hadoop102:9092,hadoop103:9092,hadoop104:9092",
"key.deserializer" -> classOf[StringDeserializer],
"value.deserializer" -> classOf[StringDeserializer],
"group.id" -> groupid,
"auto.offset.reset" -> "earliest",
"enable.auto.commit" -> (false: lang.Boolean)
)
//查询mysql是否存在偏移量
val sqlProxy = new SqlProxy()
val offsetMap = new mutable.HashMap[TopicPartition, Long]()
val client = DataSourceUtil.getConnection
try {
sqlProxy.executeQuery(client, "select *from `offset_manager` where groupid=?", Array(groupid), new QueryCallback {
override def process(rs: ResultSet): Unit = {
while (rs.next()) {
val model = new TopicPartition(rs.getString(2), rs.getInt(3))
val offset = rs.getLong(4)
offsetMap.put(model, offset)
}
rs.close()
}
})
} catch {
case e: Exception => e.printStackTrace()
} finally {
sqlProxy.shutdown(client)
}
//设置kafka消费数据的参数 判断本地是否有偏移量 有则根据偏移量继续消费 无则重新消费
val stream: InputDStream[ConsumerRecord[String, String]] = if (offsetMap.isEmpty) {
KafkaUtils.createDirectStream(
ssc, LocationStrategies.PreferConsistent, ConsumerStrategies.Subscribe[String, String](topics, kafkaMap))
} else {
KafkaUtils.createDirectStream(
ssc, LocationStrategies.PreferConsistent, ConsumerStrategies.Subscribe[String, String](topics, kafkaMap, offsetMap))
}
//解析json数据
val dsStream = stream.mapPartitions(partitions => {
partitions.map(item => {
val json = item.value()
val jsonObject = ParseJsonData.getJsonData(json)
val userId = jsonObject.getIntValue("uid")
val cwareid = jsonObject.getIntValue("cwareid")
val videoId = jsonObject.getIntValue("videoid")
val chapterId = jsonObject.getIntValue("chapterid")
val edutypeId = jsonObject.getIntValue("edutypeid")
val subjectId = jsonObject.getIntValue("subjectid")
val sourceType = jsonObject.getString("sourceType")
val speed = jsonObject.getIntValue("speed")
val ts = jsonObject.getLong("ts")
val te = jsonObject.getLong("te")
val ps = jsonObject.getIntValue("ps")
val pe = jsonObject.getIntValue("pe")
LearnModel(userId, cwareid, videoId, chapterId, edutypeId, subjectId, sourceType, speed, ts, te, ps, pe)
})
})
dsStream.foreachRDD(rdd => {
rdd.cache()
//统计播放视频 有效时长 完成时长 总时长
rdd.groupBy(item => item.userId + "_" + item.cwareId + "_" + item.videoId).foreachPartition(partitoins => {
val sqlProxy = new SqlProxy()
val client = DataSourceUtil.getConnection
try {
partitoins.foreach { case (key, iters) =>
calcVideoTime(key, iters, sqlProxy, client) //计算视频时长
}
} catch {
case e: Exception => e.printStackTrace()
} finally {
sqlProxy.shutdown(client)
}
})
//统计章节下视频播放总时长
rdd.mapPartitions(partitions => {
partitions.map(item => {
val totaltime = Math.ceil((item.te - item.ts) / 1000).toLong
val key = item.chapterId
(key, totaltime)
})
}).reduceByKey(_ + _)
.foreachPartition(partitoins => {
val sqlProxy = new SqlProxy()
val client = DataSourceUtil.getConnection
try {
partitoins.foreach(item => {
sqlProxy.executeUpdate(client, "insert into chapter_learn_detail(chapterid,totaltime) values(?,?) on duplicate key" +
" update totaltime=totaltime+?", Array(item._1, item._2, item._2))
})
} catch {
case e: Exception => e.printStackTrace()
} finally {
sqlProxy.shutdown(client)
}
})
//统计课件下的总播放时长
rdd.mapPartitions(partitions => {
partitions.map(item => {
val totaltime = Math.ceil((item.te - item.ts) / 1000).toLong
val key = item.cwareId
(key, totaltime)
})
}).reduceByKey(_ + _).foreachPartition(partitions => {
val sqlProxy = new SqlProxy()
val client = DataSourceUtil.getConnection
try {
partitions.foreach(item => {
sqlProxy.executeUpdate(client, "insert into cwareid_learn_detail(cwareid,totaltime) values(?,?) on duplicate key " +
"update totaltime=totaltime+?", Array(item._1, item._2, item._2))
})
} catch {
case e: Exception => e.printStackTrace()
} finally {
sqlProxy.shutdown(client)
}
})
//统计辅导下的总播放时长
rdd.mapPartitions(partitions => {
partitions.map(item => {
val totaltime = Math.ceil((item.te - item.ts) / 1000).toLong
val key = item.edutypeId
(key, totaltime)
})
}).reduceByKey(_ + _).foreachPartition(partitions => {
val sqlProxy = new SqlProxy()
val client = DataSourceUtil.getConnection
try {
partitions.foreach(item => {
sqlProxy.executeUpdate(client, "insert into edutype_learn_detail(edutypeid,totaltime) values(?,?) on duplicate key " +
"update totaltime=totaltime+?", Array(item._1, item._2, item._2))
})
} catch {
case e: Exception => e.printStackTrace()
} finally {
sqlProxy.shutdown(client)
}
})
//统计同一资源平台下的总播放时长
rdd.mapPartitions(partitions => {
partitions.map(item => {
val totaltime = Math.ceil((item.te - item.ts) / 1000).toLong
val key = item.sourceType
(key, totaltime)
})
}).reduceByKey(_ + _).foreachPartition(partitions => {
val sqlProxy = new SqlProxy()
val client = DataSourceUtil.getConnection
try {
partitions.foreach(item => {
sqlProxy.executeUpdate(client, "insert into sourcetype_learn_detail (sourcetype_learn,totaltime) values(?,?) on duplicate key " +
"update totaltime=totaltime+?", Array(item._1, item._2, item._2))
})
} catch {
case e: Exception => e.printStackTrace()
} finally {
sqlProxy.shutdown(client)
}
})
// 统计同一科目下的播放总时长
rdd.mapPartitions(partitions => {
partitions.map(item => {
val totaltime = Math.ceil((item.te - item.ts) / 1000).toLong
val key = item.subjectId
(key, totaltime)
})
}).reduceByKey(_ + _).foreachPartition(partitons => {
val sqlProxy = new SqlProxy()
val clinet = DataSourceUtil.getConnection
try {
partitons.foreach(item => {
sqlProxy.executeUpdate(clinet, "insert into subject_learn_detail(subjectid,totaltime) values(?,?) on duplicate key " +
"update totaltime=totaltime+?", Array(item._1, item._2, item._2))
})
} catch {
case e: Exception => e.printStackTrace()
} finally {
sqlProxy.shutdown(clinet)
}
})
})
//计算转换率
//处理完 业务逻辑后 手动提交offset维护到本地 mysql中
stream.foreachRDD(rdd => {
val sqlProxy = new SqlProxy()
val client = DataSourceUtil.getConnection
try {
val offsetRanges: Array[OffsetRange] = rdd.asInstanceOf[HasOffsetRanges].offsetRanges
for (or <- offsetRanges) {
sqlProxy.executeUpdate(client, "replace into `offset_manager` (groupid,topic,`partition`,untilOffset) values(?,?,?,?)",
Array(groupid, or.topic, or.partition.toString, or.untilOffset))
}
} catch {
case e: Exception => e.printStackTrace()
} finally {
sqlProxy.shutdown(client)
}
})
ssc.start()
ssc.awaitTermination()
}
/**
* 计算视频 有效时长 完成时长 总时长
*
* @param key
* @param iters
* @param sqlProxy
* @param client
*/
def calcVideoTime(key: String, iters: Iterable[LearnModel], sqlProxy: SqlProxy, client: Connection) = {
val keys = key.split("_")
val userId = keys(0).toInt
val cwareId = keys(1).toInt
val videoId = keys(2).toInt
//查询历史数据
var interval_history = ""
sqlProxy.executeQuery(client, "select play_interval from video_interval where userid=? and cwareid=? and videoid=?",
Array(userId, cwareId, videoId), new QueryCallback {
override def process(rs: ResultSet): Unit = {
while (rs.next()) {
interval_history = rs.getString(1)
}
rs.close()
}
})
var effective_duration_sum = 0l //有效总时长
var complete_duration_sum = 0l //完成总时长
var cumulative_duration_sum = 0l //播放总时长
val learnList = iters.toList.sortBy(item => item.ps) //转成list 并根据开始区间升序排序
learnList.foreach(item => {
if ("".equals(interval_history)) {
//没有历史区间
val play_interval = item.ps + "-" + item.pe //有效区间
val effective_duration = Math.ceil((item.te - item.ts) / 1000) //有效时长
val complete_duration = item.pe - item.ps //完成时长
effective_duration_sum += effective_duration.toLong
cumulative_duration_sum += effective_duration.toLong
complete_duration_sum += complete_duration
interval_history = play_interval
} else {
//有历史区间进行对比
val interval_arry = interval_history.split(",").sortBy(a => (a.split("-")(0).toInt, a.split("-")(1).toInt))
val tuple = getEffectiveInterval(interval_arry, item.ps, item.pe)
val complete_duration = tuple._1 //获取实际有效完成时长
val effective_duration = Math.ceil((item.te - item.ts) / 1000) / (item.pe - item.ps) * complete_duration //计算有效时长
val cumulative_duration = Math.ceil((item.te - item.ts) / 1000) //累计时长
interval_history = tuple._2
effective_duration_sum += effective_duration.toLong
complete_duration_sum += complete_duration
cumulative_duration_sum += cumulative_duration.toLong
}
sqlProxy.executeUpdate(client, "insert into video_interval(userid,cwareid,videoid,play_interval) values(?,?,?,?) " +
"on duplicate key update play_interval=?", Array(userId, cwareId, videoId, interval_history, interval_history))
sqlProxy.executeUpdate(client, "insert into video_learn_detail(userid,cwareid,videoid,totaltime,effecttime,completetime) " +
"values(?,?,?,?,?,?) on duplicate key update totaltime=totaltime+?,effecttime=effecttime+?,completetime=completetime+?",
Array(userId, cwareId, videoId, cumulative_duration_sum, effective_duration_sum, complete_duration_sum, cumulative_duration_sum,
effective_duration_sum, complete_duration_sum))
})
}
/**
* 计算有效区间
*
* @param array
* @param start
* @param end
* @return
*/
def getEffectiveInterval(array: Array[String], start: Int, end: Int) = {
var effective_duration = end - start
var bl = false //是否对有效时间进行修改
import scala.util.control.Breaks._
breakable {
for (i <- 0 until array.length) {
//循环各区间段
var historyStart = 0 //获取其中一段的开始播放区间
var historyEnd = 0 //获取其中一段结束播放区间
val item = array(i)
try {
historyStart = item.split("-")(0).toInt
historyEnd = item.split("-")(1).toInt
} catch {
case e: Exception => throw new Exception("error array:" + array.mkString(","))
}
if (start >= historyStart && historyEnd >= end) {
//已有数据占用全部播放时长 此次播放无效
effective_duration = 0
bl = true
break()
} else if (start <= historyStart && end > historyStart && end < historyEnd) {
//和已有数据左侧存在交集 扣除部分有效时间（以老数据为主进行对照）
effective_duration -= end - historyStart
array(i) = start + "-" + historyEnd
bl = true
} else if (start > historyStart && start < historyEnd && end >= historyEnd) {
//和已有数据右侧存在交集 扣除部分有效时间
effective_duration -= historyEnd - start
array(i) = historyStart + "-" + end
bl = true
} else if (start < historyStart && end > historyEnd) {
//现数据 大于旧数据 扣除旧数据所有有效时间
effective_duration -= historyEnd - historyStart
array(i) = start + "-" + end
bl = true
}
}
}
val result = bl match {
case false => {
//没有修改原array 没有交集 进行新增
val distinctArray2 = ArrayBuffer[String]()
distinctArray2.appendAll(array)
distinctArray2.append(start + "-" + end)
val distinctArray = distinctArray2.distinct.sortBy(a => (a.split("-")(0).toInt, a.split("-")(1).toInt))
val tmpArray = ArrayBuffer[String]()
tmpArray.append(distinctArray(0))
for (i <- 1 until distinctArray.length) {
val item = distinctArray(i).split("-")
val tmpItem = tmpArray(tmpArray.length - 1).split("-")
val itemStart = item(0)
val itemEnd = item(1)
val tmpItemStart = tmpItem(0)
val tmpItemEnd = tmpItem(1)
if (tmpItemStart.toInt < itemStart.toInt && tmpItemEnd.toInt < itemStart.toInt) {
//没有交集
tmpArray.append(itemStart + "-" + itemEnd)
} else {
//有交集
val resultStart = tmpItemStart
val resultEnd = if (tmpItemEnd.toInt > itemEnd.toInt) tmpItemEnd else itemEnd
tmpArray(tmpArray.length - 1) = resultStart + "-" + resultEnd
}
}
val play_interval = tmpArray.sortBy(a => (a.split("-")(0).toInt, a.split("-")(1).toInt)).mkString(",")
play_interval
}
case true => {
//修改了原array 进行区间重组
val distinctArray = array.distinct.sortBy(a => (a.split("-")(0).toInt, a.split("-")(1).toInt))
val tmpArray = ArrayBuffer[String]()
tmpArray.append(distinctArray(0))
for (i <- 1 until distinctArray.length) {
val item = distinctArray(i).split("-")
val tmpItem = tmpArray(tmpArray.length - 1).split("-")
val itemStart = item(0)
val itemEnd = item(1)
val tmpItemStart = tmpItem(0)
val tmpItemEnd = tmpItem(1)
if (tmpItemStart.toInt < itemStart.toInt && tmpItemEnd.toInt < itemStart.toInt) {
//没有交集
tmpArray.append(itemStart + "-" + itemEnd)
} else {
//有交集
val resultStart = tmpItemStart
val resultEnd = if (tmpItemEnd.toInt > itemEnd.toInt) tmpItemEnd else itemEnd
tmpArray(tmpArray.length - 1) = resultStart + "-" + resultEnd
}
}
val play_interval = tmpArray.sortBy(a => (a.split("-")(0).toInt, a.split("-")(1).toInt)).mkString(",")
play_interval
}
}
(effective_duration, result)
}
}